From 6be21acde1d22c7921c981ba8d007fe7d6a33cb0 Mon Sep 17 00:00:00 2001 From: Vladimir Eremin Date: Sat, 10 Oct 2026 20:32:12 +0300 Subject: [PATCH] feat: initialize Debian nodes through shared provisioning Add local SSH enrollment for clean Debian 13 hosts using the same baseline and user-data contract as Packer. Preserve provider networking, validate storage and repository ownership, and protect final SSH and security-service activation with persistent access recovery. Document composition-scoped replication patterns for independent node repositories and add behavioral coverage for provisioning, retries, and recovery. Native verification of the security activation transaction remains the production enrollment gate. Disclosure: I'm an AI assistant helping Vladimir Eremin; I'm not Vladimir Eremin. --- ...-packer-main-manual-build-golden-image.yml | 8 +- README.md | 48 ++ actions/configure/base.py | 72 +-- actions/init.py | 16 + ci/cloud-init/user-data.template.yaml | 53 +- ci/packer/apex-debian-13.pkr.hcl | 23 +- ci/packer/scripts/01-base-system.sh | 32 +- ci/packer/scripts/02-storage-docker.sh | 93 +-- ci/packer/scripts/03-bootstrap-helper.sh | 139 +---- ci/packer/scripts/04-cleanup.sh | 29 +- docs/rfcs/composition-replication.md | 162 +++++ engine/cli.py | 2 +- engine/provisioning/__init__.py | 3 + engine/provisioning/access.py | 558 ++++++++++++++++++ engine/provisioning/baseline.py | 412 +++++++++++++ engine/provisioning/contract.py | 236 ++++++++ engine/provisioning/image.py | 124 ++++ engine/provisioning/repository.py | 121 ++++ engine/provisioning/security.py | 475 +++++++++++++++ engine/provisioning/seed.sh | 87 +++ engine/provisioning/state.py | 128 ++++ engine/provisioning/storage.py | 508 ++++++++++++++++ engine/provisioning/target.py | 170 ++++++ engine/provisioning/transport.py | 385 ++++++++++++ engine/provisioning/userdata.py | 29 + tests/provision_fakes.py | 38 ++ tests/test_provision_access.py | 513 ++++++++++++++++ tests/test_provision_baseline.py | 338 +++++++++++ tests/test_provision_contract.py | 230 ++++++++ tests/test_provision_image.py | 37 ++ tests/test_provision_repository.py | 132 +++++ tests/test_provision_security.py | 544 +++++++++++++++++ tests/test_provision_security_access.py | 105 ++++ tests/test_provision_state.py | 80 +++ tests/test_provision_storage.py | 453 ++++++++++++++ tests/test_provision_target.py | 87 +++ tests/test_provision_transport.py | 517 ++++++++++++++++ tests/test_provision_userdata.py | 30 + 38 files changed, 6625 insertions(+), 392 deletions(-) create mode 100644 actions/init.py create mode 100644 docs/rfcs/composition-replication.md create mode 100644 engine/provisioning/__init__.py create mode 100644 engine/provisioning/access.py create mode 100644 engine/provisioning/baseline.py create mode 100644 engine/provisioning/contract.py create mode 100644 engine/provisioning/image.py create mode 100644 engine/provisioning/repository.py create mode 100644 engine/provisioning/security.py create mode 100755 engine/provisioning/seed.sh create mode 100644 engine/provisioning/state.py create mode 100644 engine/provisioning/storage.py create mode 100644 engine/provisioning/target.py create mode 100644 engine/provisioning/transport.py create mode 100644 engine/provisioning/userdata.py create mode 100644 tests/provision_fakes.py create mode 100644 tests/test_provision_access.py create mode 100644 tests/test_provision_baseline.py create mode 100644 tests/test_provision_contract.py create mode 100644 tests/test_provision_image.py create mode 100644 tests/test_provision_repository.py create mode 100644 tests/test_provision_security.py create mode 100644 tests/test_provision_security_access.py create mode 100644 tests/test_provision_state.py create mode 100644 tests/test_provision_storage.py create mode 100644 tests/test_provision_target.py create mode 100644 tests/test_provision_transport.py create mode 100644 tests/test_provision_userdata.py diff --git a/.github/workflows/ci-packer-main-manual-build-golden-image.yml b/.github/workflows/ci-packer-main-manual-build-golden-image.yml index ccd0444..024a9da 100644 --- a/.github/workflows/ci-packer-main-manual-build-golden-image.yml +++ b/.github/workflows/ci-packer-main-manual-build-golden-image.yml @@ -24,7 +24,11 @@ jobs: uses: actions/checkout@v4 - name: Validate Shell Script Syntax - run: bash -n ci/packer/scripts/*.sh + run: | + for script in ci/packer/scripts/*.sh engine/provisioning/seed.sh; do + bash -n "$script" + done + bash tests/run.sh - name: Setup Packer uses: hashicorp/setup-packer@main @@ -67,7 +71,7 @@ jobs: - name: Build Debian 13 Image with Packer run: | packer init ci/packer/apex-debian-13.pkr.hcl - packer build -var "vm_name=${{ steps.vars.outputs.image_name }}" ci/packer/apex-debian-13.pkr.hcl + packer build -var "vm_name=${{ steps.vars.outputs.image_name }}" -var "provision_revision=${{ github.sha }}" ci/packer/apex-debian-13.pkr.hcl - name: Upload Image Artifact uses: actions/upload-artifact@v4 diff --git a/README.md b/README.md index 85fcedb..2cc2ed2 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,54 @@ through the whole thing. - `git` and `python3` — standard library only, nothing to install. - A Debian-based host with Docker and `docker compose` v2. +## Initialize a provider-installed server + +The SSH initializer targets clean Debian 13 amd64 machines and uses the same +provisioning modules as Packer. Copy [the user-data template](ci/cloud-init/user-data.template.yaml), +fill in the target identity, operator public key, and independent node repository, then run: + +```sh +./apex init root@server --user-data /private/path/location.user-data.yaml +``` + +The workstation needs Python and OpenSSH. Target host trust follows your SSH +configuration; private Git repositories also need a deploy key and trusted Git +host keys in the supported user-data fields. User-data contains credentials and +belongs outside tracked files. This is a strict APEX cloud-config subset, not a +general cloud-init interpreter. + +Initialization prepares packages, the administrator, storage tiers, and a pinned +repository checkout. It preserves provider networking and machine identity. A +persistent watchdog protects the switch to key-only `adam` access on port 2222 +and activation of UFW, its Docker integration, CrowdSec, and the firewall bouncer. +Fresh firewall defaults match the image baseline; existing custom policies require +compatible configuration. Unsupported conditional SSH policies stop enrollment +before baseline setup. Recovery preserves concurrent administrator edits and +reports `recovery_pending` when cleanup needs reconciliation, keeping the watchdog +armed while attempting to restore SSH independently. +After that switch, retries use `adam@server --port 2222`; the command reports the +safe resume invocation and root-owned stage records. Existing data and divergent +checkouts are preserved and reported as conflicts. Initialization does not deploy +applications or create database clusters. + +Supported node repositories declare `APEX_ENROLLMENT_INTERFACE=1` in `node.env` +alongside their compatible `init.sh`. Missing or unsupported declarations stop +enrollment before the checkout hook runs. Results record the provisioner and +baseline revisions separately, plus the node and commons commits. + +The running Debian kernel must have matching headers available. If the provider +installed an obsolete kernel, install a supported kernel and headers and reboot +before enrollment; the initializer does not reboot the server automatically. + +Earlier isolated Debian 13 VM checks cover SSH enrollment, completed-run retries, +watchdog recovery after disconnect and reboot, safe storage handling, and two +independent Packer/cloud-init clones. The added security activation transaction +still requires its Debian VM verification before production enrollment. +Production enrollment also requires a +published node repository with a compatible, available commons pin. +The [composition replication RFC](docs/rfcs/composition-replication.md) defines how +independent node repositories can host replicas of selected compositions. + ## License BSD-2-Clause — see [`LICENSE`](LICENSE). diff --git a/actions/configure/base.py b/actions/configure/base.py index b5ecc09..67fb3c0 100644 --- a/actions/configure/base.py +++ b/actions/configure/base.py @@ -1,69 +1,13 @@ -# actions/configure/base.py -"""Port of configure/base — base packages + ufw + ufw-docker helper.""" -import os, shutil, stat, urllib.request +"""Apply the bounded shared base package, journal and UFW helper policy.""" +from pathlib import Path + from engine.descriptor import Meta METADATA = Meta(summary="Configure base system packages (curl, wget, git, rsyslog, ufw, ufw-docker).") -UFW_DOCKER_URL = "https://github.com/chaifeng/ufw-docker/raw/master/ufw-docker" + def run(ctx, args): - log, s = ctx.log, ctx.sys - log.info("Starting base system initialization...") - if not os.path.isfile("/etc/debian_version"): - log.error("Requires a Debian-based system (Debian/Ubuntu)."); raise SystemExit(1) - needed = [p for p in ["curl", "wget", "git", "rsyslog"] if not shutil.which(p)] - if needed: - log.info("Updating package lists..."); s.sudo(["apt-get", "update", "-y"]) - log.info(f"Installing missing essential packages: {', '.join(needed)}...") - s.sudo(["apt-get", "install", "-y", "-o", "DPkg::Lock::Timeout=60", *needed]) - else: - log.info("Essential packages already installed.") - s.ensure_running("rsyslog") - - # Configure APT to discard downloaded archives - apt_clean_path = "/etc/apt/apt.conf.d/99-apex" - apt_clean_content = 'APT::Keep-Downloaded-Packages "0";\n' - if os.path.exists("/etc/apt/apt.conf.d/01clean"): - s.sudo(["rm", "-f", "/etc/apt/apt.conf.d/01clean"]) - try: - current_apt = open(apt_clean_path).read() if os.path.exists(apt_clean_path) else None - except Exception: - current_apt = None - if current_apt != apt_clean_content: - log.info("Configuring APT to discard downloaded archives...") - s.sudo(["tee", apt_clean_path], input=apt_clean_content) - - # Limit systemd-journald max disk usage - journal_d = "/etc/systemd/journald.conf.d" - journal_conf = os.path.join(journal_d, "99-apex.conf") - journal_content = "[Journal]\nSystemMaxUse=200M\n" - if os.path.exists(os.path.join(journal_d, "maxuse.conf")): - s.sudo(["rm", "-f", os.path.join(journal_d, "maxuse.conf")]) - try: - current_journal = open(journal_conf).read() if os.path.exists(journal_conf) else None - except Exception: - current_journal = None - if current_journal != journal_content: - log.info("Configuring systemd-journald SystemMaxUse=200M...") - s.sudo(["mkdir", "-p", journal_d]) - s.sudo(["tee", journal_conf], input=journal_content) - s.sudo(["systemctl", "restart", "systemd-journald"]) - - if not shutil.which("ufw"): - log.info("Installing UFW..."); s.sudo(["apt-get", "install", "-y", "-o", "DPkg::Lock::Timeout=60", "ufw"]) - else: - log.info("UFW already installed.") - from engine.lib.ufw import resolve_ufw_docker_bin - binp = resolve_ufw_docker_bin() - if not (os.path.isfile(binp) and os.access(binp, os.X_OK)): - log.info("Downloading ufw-docker helper...") - target_bin = os.path.expanduser("~/.local/bin/ufw-docker") - os.makedirs(os.path.dirname(target_bin), exist_ok=True) - urllib.request.urlretrieve(UFW_DOCKER_URL, target_bin) - os.chmod(target_bin, os.stat(target_bin).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - log.success(f"ufw-docker downloaded to {target_bin}.") - else: - log.info(f"ufw-docker already exists at {binp}.") - log.info("Ensuring ufw-docker routing rules are installed in UFW...") - s.sudo([binp, "install"]) - log.success("Base system initialization completed.") + ctx.log.info("Starting base system initialization...") + commons = Path(__file__).resolve().parents[2] + ctx.sys.sudo(["python3", "-m", "engine.provisioning.baseline", "--base-policy"], cwd=str(commons)) + ctx.log.success("Base system initialization completed.") diff --git a/actions/init.py b/actions/init.py new file mode 100644 index 0000000..a032ac7 --- /dev/null +++ b/actions/init.py @@ -0,0 +1,16 @@ +"""Initialize a Debian target without resolving a workstation node identity.""" +from engine.descriptor import Arg, Meta, Opt +from engine.provisioning.transport import initialize + +METADATA = Meta(summary='Initialize Debian 13 through protected SSH enrollment.', args=[ + Arg('target', 'Bootstrap USER@HOST; root or an existing administrator with sudo -n'), + Opt('--user-data', 'Supported APEX cloud-config file'), + Opt('--port', 'Bootstrap SSH port; otherwise use SSH configuration'), + Opt('--identity-file', 'SSH private identity file; otherwise use SSH configuration/agent'), +]) + + +def run(ctx, args): + initialize(ctx.paths.commons, args.target, args.user_data, args.port, args.identity_file, + progress=ctx.log.info) + return 0 diff --git a/ci/cloud-init/user-data.template.yaml b/ci/cloud-init/user-data.template.yaml index d4ed6b9..ca2c3e4 100644 --- a/ci/cloud-init/user-data.template.yaml +++ b/ci/cloud-init/user-data.template.yaml @@ -1,52 +1,33 @@ #cloud-config -# ============================================================================== -# Cloud-Init User-Data Template for apex.ermnvldmr.com Node -# Compatible with generic cloud-init and virtualized environments (QEMU/KVM) -# -# Usage: -# 1. Copy template: cp ci/cloud-init/user-data.template.yaml user-data.yaml -# 2. Replace the public SSH key for adam. -# 3. Replace the private SSH deploy key for GitHub access. -# 4. Adjust APEX_REPO_NAME, APEX_REPO_URL, and tier devices in /etc/apex/bootstrap.env. -# 5. Pass user-data.yaml as user-data during VM creation. -# ============================================================================== - +# Use this same file for cloud-init or: ./apex init root@HOST --user-data FILE +# Replace the hostname, repository and public key. Keep filled files outside Git. +# SSH repositories additionally require approved deferred id_ed25519 (0600) +# and known_hosts (0644) writes under /home/adam/.ssh, owned by adam:adam. +# Supply trusted Git host keys; do not infer trust from ssh-keyscan alone. preserve_hostname: false -hostname: daedalus.a1.apex.ermnvldmr.com - +hostname: metis.a1.apex.ermnvldmr.com users: - name: adam sudo: ALL=(ALL) NOPASSWD:ALL groups: [sudo, docker] shell: /bin/bash ssh_authorized_keys: - - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI_REPLACE_WITH_YOUR_PUBLIC_SSH_KEY adam@workstation" - + - "ssh-ed25519 REPLACE_WITH_OPERATOR_PUBLIC_KEY operator@workstation" write_files: - # ---------------------------------------------------------------------------- - # 1. SSH Deploy Key for GitHub Submodule & Repo Access - # ---------------------------------------------------------------------------- - - path: /home/adam/.ssh/id_ed25519 - owner: adam:adam - permissions: '0600' - defer: true - content: | - -----BEGIN OPENSSH PRIVATE KEY----- - # REPLACE WITH YOUR ED25519 DEPLOY PRIVATE KEY - -----END OPENSSH PRIVATE KEY----- - - # ---------------------------------------------------------------------------- - # 2. Node Bootstrap Parameters - # ---------------------------------------------------------------------------- - path: /etc/apex/bootstrap.env owner: root:root permissions: '0600' content: | - APEX_REPO_NAME="daedalus.apex.ermnvldmr.com" - APEX_REPO_URL="git@github.com:deytenit/daedalus.apex.ermnvldmr.com.git" - APEX_TIER1_DEVICE="/dev/vdb" - APEX_TIER2_DEVICE="/dev/vdc" + APEX_REPO_NAME="metis.apex.ermnvldmr.com" + APEX_REPO_URL="https://github.com/deytenit/metis.apex.ermnvldmr.com.git" + APEX_TIER1_MODE="directory" + APEX_TIER1_DEVICE="" + APEX_TIER2_MODE="directory" + APEX_TIER2_DEVICE="" + APEX_TIER3_MODE="directory" APEX_TIER3_DEVICE="" - +# For a verified empty disk, explicitly select MODE=create and DEVICE=/dev/disk/by-id/... +# For an existing single-device pool, select MODE=import, DEVICE and POOL_GUID. +# Occupied devices are never erased; missing devices never fall back to directories. runcmd: - /usr/local/bin/apex-bootstrap diff --git a/ci/packer/apex-debian-13.pkr.hcl b/ci/packer/apex-debian-13.pkr.hcl index ca2cfd2..b36b8e0 100644 --- a/ci/packer/apex-debian-13.pkr.hcl +++ b/ci/packer/apex-debian-13.pkr.hcl @@ -27,6 +27,11 @@ variable "accelerator" { default = "kvm" } +variable "provision_revision" { + type = string + default = "working-tree" +} + source "qemu" "apex_debian" { iso_url = var.debian_cloud_image_url iso_checksum = var.debian_cloud_image_checksum @@ -34,7 +39,7 @@ source "qemu" "apex_debian" { disk_size = "10G" format = "qcow2" output_directory = "output-qemu" - shutdown_command = "echo 'packer' | sudo -S shutdown -P now" + shutdown_command = "sudo -n shutdown -P now" accelerator = var.accelerator cd_files = ["ci/packer/cidata/user-data", "ci/packer/cidata/meta-data"] cd_label = "cidata" @@ -55,8 +60,22 @@ source "qemu" "apex_debian" { build { sources = ["source.qemu.apex_debian"] + provisioner "file" { + source = "engine" + destination = "/tmp/" + } + provisioner "shell" { - execute_command = "echo 'packer' | sudo -S sh -c '{{ .Vars }} {{ .Path }}'" + execute_command = "sudo -n sh -c '{{ .Vars }} {{ .Path }}'" + environment_vars = ["APEX_PROVISION_REVISION=${var.provision_revision}"] + inline = [ + "bash /tmp/engine/provisioning/seed.sh", + "cd /tmp && python3 -m engine.provisioning.image install" + ] + } + + provisioner "shell" { + execute_command = "sudo -n sh -c '{{ .Vars }} {{ .Path }}'" scripts = [ "ci/packer/scripts/01-base-system.sh", "ci/packer/scripts/02-storage-docker.sh", diff --git a/ci/packer/scripts/01-base-system.sh b/ci/packer/scripts/01-base-system.sh index c0b829a..f7efb62 100755 --- a/ci/packer/scripts/01-base-system.sh +++ b/ci/packer/scripts/01-base-system.sh @@ -1,31 +1,8 @@ #!/usr/bin/env bash set -euo pipefail export DEBIAN_FRONTEND=noninteractive - -echo "=== [1/6] Configuring Base Packages ===" -apt-get update -apt-get install -y --no-install-recommends \ - curl wget tcpdump socat rsync bind9-dnsutils iproute2 net-tools traceroute ca-certificates \ - jq sed gawk tar gzip bzip2 xz-utils unzip htop iotop strace lsof sysstat \ - parted fdisk pciutils usbutils smartmontools tmux vim less tree bash-completion \ - git python3 python3-minimal sudo rsyslog cron qemu-guest-agent - -# Prevent apt from retaining downloaded archives on disk -echo 'APT::Keep-Downloaded-Packages "0";' > /etc/apt/apt.conf.d/99-apex - -# Limit systemd journal disk usage -mkdir -p /etc/systemd/journald.conf.d -cat << 'EOF_JOURNAL' > /etc/systemd/journald.conf.d/99-apex.conf -[Journal] -SystemMaxUse=200M -EOF_JOURNAL - -echo "=== [2/6] Configuring Sudo & Passwordless Escalation ===" -cat << 'EOF_SUDO' > /etc/sudoers.d/99-apex -%sudo ALL=(ALL:ALL) NOPASSWD: ALL -EOF_SUDO -chmod 0440 /etc/sudoers.d/99-apex -visudo -cf /etc/sudoers.d/99-apex +apt-get -o DPkg::Lock::Timeout=60 update +apt-get -o DPkg::Lock::Timeout=60 install -y --no-upgrade --no-install-recommends qemu-guest-agent cloud-init echo "=== [3/6] Configuring VirtIO Kernel Modules ===" cat << 'EOF_VIRTIO' > /etc/modules-load.d/virtio.conf @@ -46,11 +23,6 @@ fi update-grub mkdir -p /etc/systemd/system/serial-getty@ttyS0.service.d -cat << 'EOF_GETTY' > /etc/systemd/system/serial-getty@ttyS0.service.d/autologin.conf -[Service] -ExecStart= -ExecStart=-/sbin/agetty --autologin root --noclear --keep-baud 115200,38400,9600 %I $TERM -EOF_GETTY systemctl enable serial-getty@ttyS0.service || true systemctl enable qemu-guest-agent 2>/dev/null || true diff --git a/ci/packer/scripts/02-storage-docker.sh b/ci/packer/scripts/02-storage-docker.sh index 43ee42a..e8c0175 100755 --- a/ci/packer/scripts/02-storage-docker.sh +++ b/ci/packer/scripts/02-storage-docker.sh @@ -1,94 +1,3 @@ #!/usr/bin/env bash set -euo pipefail -export DEBIAN_FRONTEND=noninteractive - -echo "=== [1/5] Configuring Debian Repositories for OpenZFS DKMS ===" -# Enable contrib and non-free-firmware for zfs-dkms -sed -i 's/Components: main/Components: main contrib non-free-firmware/g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \ -sed -i 's/main$/main contrib non-free-firmware/g' /etc/apt/sources.list 2>/dev/null || true - -apt-get update -apt-get install -y --no-install-recommends linux-headers-amd64 zfsutils-linux zfs-dkms - -# Pre-compile DKMS modules during bake time so first boot has zero compilation delay -systemctl enable zfs-import-cache zfs-import.target zfs.target || true - -echo "=== [2/5] Installing Docker CE from Official Debian Repository ===" -for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do - apt-get remove -y "$pkg" 2>/dev/null || true -done - -install -m 0755 -d /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc -chmod a+r /etc/apt/keyrings/docker.asc - -echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - tee /etc/apt/sources.list.d/docker.list > /dev/null - -apt-get update -apt-get install -y --no-install-recommends \ - docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - -mkdir -p /srv/docker -cat << 'EOF_DOCKER' > /etc/docker/daemon.json -{ - "data-root": "/srv/docker", - "log-driver": "json-file", - "log-opts": { - "max-size": "20m", - "max-file": "3" - } -} -EOF_DOCKER -systemctl enable docker.service containerd.service - -echo "=== [3/5] Installing and Configuring UFW + ufw-docker ===" -apt-get install -y --no-install-recommends ufw iptables - -curl -fsSL https://github.com/chaifeng/ufw-docker/raw/master/ufw-docker -o /usr/local/bin/ufw-docker -chmod +x /usr/local/bin/ufw-docker - -# Pre-configure firewall baseline (keep port 22 open during build for Packer SSH session) -ufw default deny incoming -ufw default allow outgoing -ufw default allow routed -ufw allow 22/tcp comment 'SSH default for builder' -ufw allow 2222/tcp comment 'SSH for Apex' -ufw --force enable - -# Set up ufw-docker routing in after.rules (now that UFW is active) -/usr/local/bin/ufw-docker install || true - -# Symlink into adam ~/.local/bin for engine compatibility -mkdir -p /etc/skel/.local/bin -ln -sf /usr/local/bin/ufw-docker /etc/skel/.local/bin/ufw-docker - -echo "=== [4/5] Hardening SSH Configuration ===" -mkdir -p /etc/ssh/sshd_config.d -cat << 'EOF_SSH' > /etc/ssh/sshd_config.d/99-apex.conf -Port 2222 -PasswordAuthentication no -PermitEmptyPasswords no -KbdInteractiveAuthentication no -X11Forwarding no -AllowUsers adam -EOF_SSH - -echo "=== [5/5] Installing CrowdSec ===" -# PackageCloud does not publish a dedicated 'trixie' suite yet; bookworm suite works seamlessly on Debian 13 -export os=debian -export dist=bookworm -if curl -fsSL https://packagecloud.io/install/repositories/crowdsec/crowdsec/script.deb.sh | bash; then - apt-get update - apt-get install -y --no-install-recommends crowdsec crowdsec-firewall-bouncer-iptables -else - echo "PackageCloud script failed, falling back to Debian native repository..." - rm -f /etc/apt/sources.list.d/crowdsec*.list - apt-get update - apt-get install -y --no-install-recommends crowdsec crowdsec-firewall-bouncer-iptables || \ - apt-get install -y --no-install-recommends crowdsec -fi - -echo "=== Docker, ZFS & Security Stack Complete ===" +/usr/local/bin/apex-provision-image baseline diff --git a/ci/packer/scripts/03-bootstrap-helper.sh b/ci/packer/scripts/03-bootstrap-helper.sh index 56dc391..5f1dd11 100755 --- a/ci/packer/scripts/03-bootstrap-helper.sh +++ b/ci/packer/scripts/03-bootstrap-helper.sh @@ -1,140 +1,3 @@ #!/usr/bin/env bash set -euo pipefail - -echo "=== Baking /usr/local/bin/apex-bootstrap Runner ===" - -cat << 'BOOTSTRAP_EOF' > /usr/local/bin/apex-bootstrap -#!/usr/bin/env bash -set -euo pipefail - -MARKER="/var/lib/apex-bootstrapped" -if [ -f "$MARKER" ]; then - echo "[apex-bootstrap] Node has already been bootstrapped ($MARKER exists). Exiting." - exit 0 -fi - -echo "============================================================" -echo " Starting apex.ermnvldmr.com Node Bootstrap " -echo "============================================================" - -CONFIG_ENV="/etc/apex/bootstrap.env" -if [ ! -f "$CONFIG_ENV" ]; then - echo "[apex-bootstrap] ERROR: $CONFIG_ENV not found!" >&2 - echo "[apex-bootstrap] Provide APEX_REPO_NAME, APEX_REPO_URL via cloud-init write_files." >&2 - exit 1 -fi - -source "$CONFIG_ENV" - -if [ -z "${APEX_REPO_NAME:-}" ] || [ -z "${APEX_REPO_URL:-}" ]; then - echo "[apex-bootstrap] ERROR: APEX_REPO_NAME or APEX_REPO_URL is empty in $CONFIG_ENV" >&2 - exit 1 -fi - -echo "[apex-bootstrap] Target repository: $APEX_REPO_NAME ($APEX_REPO_URL)" - -# 1. Dynamic Swap Allocation -if ! swapon --show | grep -q "/swapfile"; then - echo "[apex-bootstrap] Configuring dynamic swapfile..." - TOTAL_MEM_KB=$(grep MemTotal /proc/meminfo | awk '{print $2}') - ROOT_AVAIL_KB=$(df -k / | awk 'NR==2 {print $4}') - SWAP_SIZE_MB=2048 - if [ "$TOTAL_MEM_KB" -gt 4194304 ]; then - SWAP_SIZE_MB=4096 - fi - if [ "$ROOT_AVAIL_KB" -gt $((SWAP_SIZE_MB * 1024 * 2)) ]; then - echo "[apex-bootstrap] Allocating ${SWAP_SIZE_MB}MB swap..." - fallocate -l "${SWAP_SIZE_MB}M" /swapfile || dd if=/dev/zero of=/swapfile bs=1M count="$SWAP_SIZE_MB" - chmod 0600 /swapfile - mkswap /swapfile - swapon /swapfile - if ! grep -q "/swapfile" /etc/fstab; then - echo "/swapfile none swap sw 0 0" >> /etc/fstab - fi - else - echo "[apex-bootstrap] WARNING: Insufficient free space on rootfs for swapfile. Skipping swap." - fi -fi - -# 2. Storage Tiers & ZFS Configuration -mkdir -p /srv -for TIER_NUM in 1 2 3; do - TIER_PATH="/srv/tier-${TIER_NUM}.${APEX_REPO_NAME}" - DEV_VAR="APEX_TIER${TIER_NUM}_DEVICE" - DEV="${!DEV_VAR:-}" - - if [ -n "$DEV" ] && [ -b "$DEV" ]; then - echo "[apex-bootstrap] Checking mapped tier $TIER_NUM on device: $DEV" - POOL_NAME="tier-${TIER_NUM}" - - # Check if pool already exists on the device or in zfs - if zpool list -H -o name 2>/dev/null | grep -qw "$POOL_NAME"; then - echo "[apex-bootstrap] ZFS pool $POOL_NAME is already imported." - elif zpool import 2>/dev/null | grep -qw "$POOL_NAME"; then - echo "[apex-bootstrap] Importing existing ZFS pool $POOL_NAME..." - zpool import -f "$POOL_NAME" - else - echo "[apex-bootstrap] Creating new ZFS pool $POOL_NAME on $DEV..." - zpool create -f -o ashift=12 \ - -O compression=lz4 \ - -O atime=off \ - -O mountpoint="$TIER_PATH" \ - "$POOL_NAME" "$DEV" - fi - zfs set mountpoint="$TIER_PATH" "$POOL_NAME" || true - else - echo "[apex-bootstrap] Tier $TIER_NUM device unmapped. Using rootfs directory: $TIER_PATH" - mkdir -p "$TIER_PATH" - fi -done - -# 3. User Directories & Permissions -mkdir -p /srv/docker -mkdir -p /home/adam/.local/bin /home/adam/.ssh -ln -sf /usr/local/bin/ufw-docker /home/adam/.local/bin/ufw-docker - -echo "[apex-bootstrap] Setting permissions on /srv and home directory..." -chown -R adam:adam /srv /home/adam/.local /home/adam/.ssh -chmod 0700 /home/adam/.ssh -if [ -f /home/adam/.ssh/id_ed25519 ]; then - chmod 0600 /home/adam/.ssh/id_ed25519 -fi - -# 4. SSH Host Key Verification for GitHub -echo "[apex-bootstrap] Registering GitHub SSH host keys..." -ssh-keyscan -t ed25519,ecdsa,rsa github.com >> /home/adam/.ssh/known_hosts 2>/dev/null || true -chown adam:adam /home/adam/.ssh/known_hosts -chmod 0644 /home/adam/.ssh/known_hosts - -# 5. Clone Repository -REPO_DIR="/srv/${APEX_REPO_NAME}" -if [ ! -d "$REPO_DIR/.git" ]; then - echo "[apex-bootstrap] Cloning repository $APEX_REPO_URL as user adam..." - sudo -u adam git clone "$APEX_REPO_URL" "$REPO_DIR" -else - echo "[apex-bootstrap] Repository already cloned at $REPO_DIR." -fi - -# 6. Execute ./init.sh -INIT_SCRIPT="$REPO_DIR/init.sh" -if [ -f "$INIT_SCRIPT" ]; then - echo "[apex-bootstrap] Running ./init.sh from $REPO_DIR as user adam..." - sudo -u adam bash -c "cd '$REPO_DIR' && bash ./init.sh" - echo "[apex-bootstrap] ./init.sh completed successfully." -else - echo "[apex-bootstrap] No ./init.sh found in $REPO_DIR. Skipping." -fi - -# 7. Complete & Mark Bootstrapped -touch "$MARKER" -echo "============================================================" -echo " [apex-bootstrap] Node bootstrap finished successfully! " -echo " Next steps: SSH as adam on port 2222, fill secrets & run: " -echo " apex tiers/link " -echo " apex tiers/useradd && apex tiers/chown " -echo " apex configure && apex compose up " -echo "============================================================" -BOOTSTRAP_EOF - -chmod 0755 /usr/local/bin/apex-bootstrap -echo "=== /usr/local/bin/apex-bootstrap baked successfully ===" +/usr/local/bin/apex-provision-image install-runner diff --git a/ci/packer/scripts/04-cleanup.sh b/ci/packer/scripts/04-cleanup.sh index 12e5c3d..e01afe1 100755 --- a/ci/packer/scripts/04-cleanup.sh +++ b/ci/packer/scripts/04-cleanup.sh @@ -1,31 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -export DEBIAN_FRONTEND=noninteractive - -echo "=== [1/4] Purging APT Caches and Temp Files ===" -apt-get autoremove -y --purge +/usr/local/bin/apex-provision-image sanitize apt-get clean -rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -echo "=== [2/4] Resetting Machine Identity for Clones ===" -truncate -s 0 /etc/machine-id -rm -f /var/lib/dbus/machine-id -ln -sf /etc/machine-id /var/lib/dbus/machine-id - -mkdir -p /etc/systemd/system/ssh.service.d -cat << 'EOF_SSH_HOST' > /etc/systemd/system/ssh.service.d/10-generate-host-keys.conf -[Service] -ExecStartPre=-/usr/bin/ssh-keygen -A -EOF_SSH_HOST -ssh-keygen -A - -echo "=== [3/4] Resetting Cloud-Init State & Bash History ===" -ufw delete allow 22/tcp 2>/dev/null || true -cloud-init clean --logs --seed || true -rm -rf /var/lib/cloud/* -rm -f /root/.bash_history /home/*/.bash_history - -echo "=== [4/4] Syncing Filesystems ===" +cloud-init clean --logs --seed sync - -echo "=== Image Sanitization Complete ===" diff --git a/docs/rfcs/composition-replication.md b/docs/rfcs/composition-replication.md new file mode 100644 index 0000000..9206e89 --- /dev/null +++ b/docs/rfcs/composition-replication.md @@ -0,0 +1,162 @@ +# RFC Reusable composition replication patterns + +Status: Accepted architecture. The initial SSO rollout remains pending compatibility, routing, and failure verification; this document does not claim achieved high availability. + +## Purpose and decision + +APEX should make it repeatable to deploy another instance of an individual composition on another node and verify its declared failure behavior. Nodes and repositories always remain distinct. Applications are mostly unrelated, and duplicating configuration or running dedicated supporting services is acceptable when it preserves understandable ownership. + +The unit of replication is a logical application deployment, represented by separately owned Compose projects on participating nodes. Replication does not mean cloning a node, synchronizing repositories, sharing every composition, or applying one storage mechanism to every application. + +Adopt a small catalog of replication patterns, a common deployment contract, and a common verification workflow. Use application and database native mechanisms for data replication and leader election. Shared backend clusters are optional. SSO is the first worked example, not the architecture imposed on later applications. + +## Existing foundations + +APEX is a Python standard-library engine with node-local actions and handwritten Compose files. Each node repository pins commons and owns its compositions, configuration, and proprietary actions. Keep these characteristics. + +`node.env` declares node identity and a node-local `APEX_SUBNET`. The core composition creates a Docker bridge on that subnet. The checked-in fleet convention assigns separate /24 networks to nodes within a larger cluster address block. Existing firewall and routing configuration includes Xray transparent-proxy rules; this is not evidence that arbitrary replication protocols already work across every node. + +`apex compose` currently orchestrates local projects, with the core first. The action overlay provides a natural extension point for application-specific operations. Tier storage separates runtime state and secrets from tracked Compose definitions. + +## Ownership and terminology + +A **node** is an independently managed server with its own repository, identity, addresses, storage, credentials, and lifecycle. + +A **composition instance** is a local Compose project. Its repository owns its complete definition, even when another repository owns a similar project. + +A **replication group** identifies composition instances providing the same logical application. One node can participate in several unrelated groups, each with different members, roles, backend choices, and availability guarantees. A Compose project name alone does not establish group membership. + +A **replication domain** identifies the permitted network scope for a group. The user referred to this scope as Na. This RFC treats it as a set of mutually reachable participating nodes rather than requiring a shared layer-2 network. The label is independent of the engine's existing node-local `APEX_SUBNET`; no address allocations or cross-cluster membership are implied. + +A **failure domain** identifies what can disappear together, such as a physical site. Distinct node names or subnet labels are not sufficient evidence of independent failures. + +A **backend group** provides a data service to one or more application groups. It has its own members and failure contract. It can be dedicated to one application or deliberately shared by compatible applications. + +## Supported patterns + +| Pattern | Suitable application | Replication and failover responsibility | +| --- | --- | --- | +| Stateless copies | Instances can serve concurrently without unique mutable local state | Deploy equivalent configuration; route to ready instances | +| Copies using replicated backends | Application supports concurrent instances and external data services | Application instances share logical state through compatible replicated backends; backends own consistency and failover | +| Application-native cluster | Application provides supported membership, replication, and recovery | Follow its native topology and lifecycle; expose readiness and recovery through APEX conventions | +| Single active instance with standby | Application requires exclusive ownership of mutable state | Use a supported consistent state-transfer method and proven fencing before automatic promotion | +| Recovery-only deployment | Application has no validated safe live-replication arrangement | Reproducible restore and explicit activation; report that automatic availability is unsupported | + +Every service can be assessed through this catalog. Not every service can safely provide concurrent active copies. Supporting an arbitrary service means allowing an appropriate adapter or documented recovery pattern, not promising automatic active-active operation for unmodified software. + +A composition can combine patterns: stateless HTTP instances, a replicated SQL backend, and a singleton scheduled worker. Starting duplicate workers or schedulers is a separate correctness decision from starting duplicate web containers. + +Do not synchronize live database directories as a generic replication mechanism. Applications using SQLite require an application-supported solution or exclusive-writer failover; placing their database on a network filesystem is not a universal fix. [SQLite network storage guidance](https://www.sqlite.org/useovernet.html) + +## Required deployment contract + +Each participating repository must record the following non-secret information alongside the composition using the project's established configuration and documentation conventions. This RFC does not introduce a new manifest parser or require generated Compose files. + +| Contract element | Required content | +| --- | --- | +| Identity | Logical group name, local instance identity, replication domain, failure domain, topology revision | +| Members | Explicit peer identities, stable reachable endpoints, and desired membership roles | +| Pattern | Selected replication pattern and supported application/backend versions | +| State inventory | Every persistent volume, uploaded file store, session store, user directory, queue, and mutable configuration source | +| State ownership | Replication method, write authority, consistency limits, and recovery procedure for each state class | +| Dependencies | Named backend groups, access requirements, and the failure assumptions of each dependency | +| Configuration | Required common settings, intentionally local settings, secret references, and update procedure | +| Traffic | Public and internal client endpoints, readiness conditions, routing and withdrawal behavior | +| Lifecycle | Initial creation, joining, draining, upgrade order, departure, rejoin, and full-outage recovery | +| Guarantees | Maximum recovery time target, acknowledged-data-loss policy, session behavior, and supported failure scenarios | + +Desired roles must not overwrite runtime decisions. For example, a replica promoted by Sentinel must not be forcibly returned to an obsolete role on every ordinary restart. + +Repositories can adopt application changes independently only within the documented version-compatibility window. Changes to group membership, shared schemas, or wire protocols require coordinated sequencing. Planned membership changes must validate agreement among the participating reachable nodes against the native system's authoritative membership. Removing a failed member follows the native recovery procedure and must not require that failed member to acknowledge the change. + +Keep private keys, credentials, password hashes, and application encryption secrets outside tracked configuration. State explicitly which application secrets must match across replicas; node identity and node-specific credentials remain distinct. + +## APEX responsibilities + +APEX should provide reusable pattern documentation, preflight checks, composition-scoped lifecycle entry points, and a consistent way to inspect readiness and replication health. A future implementation should use the existing action and overlay model. Exact new command names and machine-readable schemas belong to the implementation design after this RFC is accepted. + +Actions operate on explicitly selected local compositions. Optional SSH invocation transports the same operations to named nodes; it does not create a fleet-wide desired-state controller. APEX must not make automatic quorum decisions or promote database primaries based on its own ping checks. + +Checks must distinguish running containers from a service that can safely serve traffic. They must report incompatible versions, unreachable required peers, mismatched membership, missing configuration, unsafe state-sharing arrangements, and absent capacity for the declared failure target. + +Shared helpers may cover common tasks. Native backend tools retain ownership of elections, replication, fencing, and recovery. Application-specific behavior stays in a bounded adapter or runbook rather than an expanding set of SSO conditionals in the engine. + +## Networking and routing + +Reuse an existing private transport when tests demonstrate that it supports the application's actual protocols. Validate advertised addresses, bidirectional connectivity, required TCP and UDP ports, reconnect behavior, and relevant MTU constraints. Do not silently replace the fleet's routing with a new overlay. + +Preserve distinct node-local bridge networks and addresses. Docker service names and host-local container IP assumptions must not be mistaken for cross-host service discovery. Peer endpoints must be explicitly reachable from the consumers that use them. + +A logical application endpoint routes only to ready instances. Dependency failure and loss of write authority must affect readiness. Both browser-facing endpoints and internal application calls must survive the same location-loss scenario. A single reverse proxy on one participating site must not be the only traffic entry point. + +Plain round-robin DNS distributes addresses but does not supply application health checks. DNS-based failover also needs its caching behavior included in the measured recovery time. [DNS load balancing](https://www.cloudflare.com/learning/performance/what-is-dns-load-balancing/) + +Automatic failover must keep working when the operator laptop and APEX CLI are absent. Its runtime mechanisms belong to the deployed services and traffic infrastructure. + +## Shared and dedicated backends + +Allow both arrangements. Prefer dedicated resources when versions, retention, durability, eviction policy, performance, or trust boundaries differ. Reuse an existing compatible cluster when doing so materially simplifies operations and its shared failure impact is accepted. + +Several applications can use separate databases and credentials within one MariaDB cluster; each does not automatically require a new Galera installation. Applications requiring PostgreSQL need an appropriate PostgreSQL solution instead. Shared database credentials are not an acceptable shortcut. + +Caches, sessions, and durable queues must not be combined solely because all speak the Redis protocol. Separate Valkey replication groups remain valid, and Sentinel can monitor multiple named groups. A Sentinel voting set is not a universal witness for unrelated database technologies. [Valkey Sentinel](https://valkey.io/topics/sentinel/) + +Application placement and backend placement must satisfy the whole dependency graph. Two application copies are not location-independent when both require an unreplicated database on one of those locations. + +## Lifecycle and failure behavior + +Initial creation and joining are different operations. Joining must verify the group identity and synchronize from the authorized source. Existing non-empty state is not overwritten implicitly. Formatting storage, discarding divergent state, and forced cluster recovery are explicit operations. + +Ordinary deployment must not bootstrap a new database cluster when peers are unreachable. A returning member catches up and passes readiness checks before it receives traffic. Repeated operations should converge or stop with an actionable diagnosis rather than silently forming a second cluster. + +Upgrade one failure domain at a time where the native system supports it. Verify redundancy and capacity before removing a member. Schema migration ownership and compatible mixed-version operation must be documented per application; reverting a container image does not automatically reverse a database migration. + +Automatic promotion of an exclusive writer requires a native election mechanism or fencing that prevents the previous writer from continuing. If this prerequisite is absent, classify promotion as manual rather than claiming safe automatic failover. + +Backups remain independent of replication and cover accidental deletion, corruption, and total group loss. Restoring a backup must not introduce an additional active writer into an existing group without its join procedure. + +## SSO first application + +Daedalus and Metis are separate repositories and application locations. Icarus is the proposed independent location for backend voting services. Other compositions on those nodes do not become replicas by association. + +The candidate SSO arrangement has two Authelia/nginx deployments and replicated persistent and session state. MariaDB/Galera can use two full data members plus an Icarus arbitrator. Valkey uses a native replication topology with three Sentinel voters. These may be dedicated compositions or reusable backend groups; SSO does not require a global SQL/cache platform for every APEX application. + +The Galera arbitrator provides voting without another SQL data copy. The remaining data member and arbitrator can preserve quorum after one data location fails. WAN latency and stable connectivity must be checked before choosing this topology. [MariaDB deployment variants](https://mariadb.com/docs/galera-cluster/galera-architecture/galera-cluster-deployment-variants/) + +The existing users source is a manually maintained YAML file. Identical read-only distribution is a possible experiment, but Authelia does not officially endorse that workaround for HA. Production acceptance requires either an explicitly accepted limitation or a supported user-directory design with its own failure coverage. Migrating passwords must preserve usable credentials or have an approved reset procedure; changing the backend is not assumed to convert hashes automatically. [Authelia stateful dependencies](https://www.authelia.com/overview/authorization/statelessness/) + +Valkey replication is asynchronous and can lose acknowledged recent changes. The proposed SSO pattern therefore cannot claim seamless preservation of every session or revocation. Its production profile must state and test the resulting security and user-visible behavior. A requirement for lossless session state requires a different validated arrangement, not a stronger label on Sentinel. + +The starting recovery target is automatic restoration of authentication within 60 seconds after loss of any one participating location. This is a proposed acceptance target, not an achieved guarantee. Session behavior is assessed separately. All SSO dependencies, including user data, required notifications, internal authentication routes, and public routing, must be included before declaring that target met. + +## Adding the next application + +1. Select the compositions and target nodes; keep node identities and unrelated projects untouched. +2. Inventory mutable state, background jobs, dependencies, and application-supported concurrency. +3. Select a pattern. Reuse compatible infrastructure or provision dedicated supporting groups deliberately. +4. Add the local composition and deployment contract to each participating repository. Provide configuration, secrets, and reachable endpoints. +5. Run preflight checks, seed or join through the native mechanism, and confirm synchronization. +6. Add the ready instance to traffic routing. +7. Execute the failure suite and record the measured guarantee before describing the application as highly available. + +A stateless dashboard should follow these steps without database machinery. A SQL-backed application may reuse a cluster. A SQLite-backed application may need a standby arrangement or remain recovery-only. This range is intentional for a fleet of unrelated applications. + +## Acceptance and rollout + +Validate the framework against at least a stateless composition and the SSO stateful example before generalizing helpers further. Do not migrate unrelated existing applications as part of the first implementation. + +For each claimed automatic-availability profile, test loss of every location individually, process failure, network partition, stale/rejoining members, and a rolling update. Observe end-user operations and dependency health, not merely container exit codes. Test without the operator laptop participating. + +Record recovery duration, acknowledged data lost, session behavior, whether conflicting writers existed, and whether manual intervention was needed. Full-group loss and backup restoration have a separate recovery test and do not inherit the single-location availability promise. + +A failed prerequisite must produce an honest unsupported or degraded result. The word replicated alone must not imply the availability target has been verified. + +## Relationship to remote initialization + +Remote initialization remains a separate companion design: a local `apex init` would prepare a Debian server using the same versioned baseline and supported enrollment inputs as the golden-image/cloud-init path. It ends before application state migration or replication-group creation. Replication recipes must not format disks or initialize clusters as a side effect of preparing a host. + +## Alternatives considered + +Whole-node/repository mirroring violates the ownership model and couples unrelated applications. A mandatory shared database platform couples applications that may need different engines or policies. A universal volume-sync flag cannot supply arbitrary application consistency. A fleet scheduler is a possible future infrastructure choice, but introducing one is unnecessary for this RFC and would not itself solve application data replication. + +The proposed pattern catalog preserves independent repositories, permits useful redundancy, and makes later composition replicas follow a repeatable, testable process. diff --git a/engine/cli.py b/engine/cli.py index 88886b4..71b9643 100644 --- a/engine/cli.py +++ b/engine/cli.py @@ -47,7 +47,7 @@ def main(argv) -> int: sys.stderr.write(f"[global] [core] [ERROR] Unknown action: {name}\n") return 1 - node, warns = (resolve(socket.getfqdn(), repo_root) if is_node else (None, [])) + node, warns = (resolve(socket.getfqdn(), repo_root) if is_node and name != 'init' else (None, [])) ctx = Context(node, Paths(repo_root, commons_dir), name, shadowed=r.shadowed) for w in warns: ctx.log.warn(w) diff --git a/engine/provisioning/__init__.py b/engine/provisioning/__init__.py new file mode 100644 index 0000000..49104dd --- /dev/null +++ b/engine/provisioning/__init__.py @@ -0,0 +1,3 @@ +"""Shared initialization primitives; importing this package performs no host changes.""" + +ENROLLMENT_INTERFACE = 1 diff --git a/engine/provisioning/access.py b/engine/provisioning/access.py new file mode 100644 index 0000000..c9997aa --- /dev/null +++ b/engine/provisioning/access.py @@ -0,0 +1,558 @@ +"""Protected SSH handover with a separately locked, persistent rollback timer.""" +from __future__ import annotations + +import argparse +import base64 +import fcntl +import fnmatch +import hashlib +import ipaddress +import json +import os +from pathlib import Path +import pwd +import re +import shlex +import stat +import subprocess +import sys +import tempfile +import time + + +class AccessError(RuntimeError): + pass + + +def include_arguments(value): + arguments, word, quote, started = [], [], None, False + for character in value: + if character == '\\': + raise AccessError('Escaped SSH Include paths require manual reconciliation') + if quote: + if character == quote: + quote = None + else: + word.append(character) + elif character in ('"', "'"): + quote, started = character, True + elif character in ' \t\r': + if started: + arguments.append(''.join(word)) + word, started = [], False + elif character == '#' and not started: + break + else: + word.append(character) + started = True + if quote: + raise AccessError('Unclosed SSH Include quote requires manual reconciliation') + if started: + arguments.append(''.join(word)) + if not arguments or len(arguments) > 128 or any(not item or len(item) > 4096 for item in arguments): + raise AccessError('Unsupported SSH Include paths require manual reconciliation') + return arguments + + +class Host: + def __init__(self, root=Path('/'), runner=subprocess.run): + self.root = Path(root).resolve() + self.runner = runner + + def path(self, path): + return self.root / path.lstrip('/') + + def command(self, args, check=True): + result = self.runner(args, capture_output=True, text=True, timeout=60) + if check and result.returncode: + raise AccessError('Access command failed: ' + args[0]) + return result + + def boot_id(self): + return self.path('/proc/sys/kernel/random/boot_id').read_text().strip() + + def fingerprint(self): + digest = hashlib.sha256() + paths = [self.path('/etc/ssh/sshd_config'), self.path('/etc/default/ssh')] + for directory in ('/etc/ssh/sshd_config.d', '/etc/systemd/system/ssh.service.d', + '/run/systemd/system/ssh.service.d', '/usr/lib/systemd/system/ssh.service.d', + '/lib/systemd/system/ssh.service.d'): + paths += sorted(self.path(directory).glob('*.conf')) + for path in paths: + digest.update(str(path.relative_to(self.root)).encode()) + digest.update(path.read_bytes() if path.exists() else b'') + return digest.hexdigest() + + def unconditional_configuration(self): + remaining, directory_entries, visited = 1024 * 1024, 0, set() + + def expand(pattern): + nonlocal directory_entries + if any(token in pattern for token in ('[:', '[.', '[=', '[^')): + raise AccessError('Extended SSH Include glob syntax requires manual reconciliation') + absolute = pattern if pattern.startswith('/') else '/etc/ssh/' + pattern + mapped = self.path(absolute) + if len(mapped.parts) > 64: + raise AccessError('SSH Include path exceeds supported limits; manual reconciliation required') + candidates = [Path(mapped.anchor)] + for component in mapped.parts[1:]: + expanded = [] + for parent in candidates: + if not any(character in component for character in '*?['): + candidate = parent / component + if os.path.lexists(candidate): + expanded.append(candidate) + continue + try: + with os.scandir(parent) as entries: + for entry in entries: + directory_entries += 1 + if directory_entries > 4096: + raise AccessError('SSH Include traversal exceeds supported limits; manual reconciliation required') + if entry.name.startswith('.') and not component.startswith('.'): + continue + if fnmatch.fnmatchcase(entry.name, component): + expanded.append(Path(entry.path)) + except (FileNotFoundError, NotADirectoryError): + continue + if len(expanded) > 128: + raise AccessError('SSH Include expansion exceeds supported limits; manual reconciliation required') + candidates = expanded + return sorted(candidates) + + def inspect(path, ancestors): + nonlocal remaining + information = path.stat() + identity = (information.st_dev, information.st_ino) + if identity in ancestors or len(ancestors) >= 16: + raise AccessError('Recursive SSH Include configuration requires manual reconciliation') + if identity in visited: + return + if len(visited) >= 128 or not stat.S_ISREG(information.st_mode) or information.st_size > remaining: + raise AccessError('SSH configuration exceeds supported inspection limits; manual reconciliation required') + visited.add(identity) + with path.open('rb') as stream: + content = stream.read(remaining + 1) + remaining -= len(content) + if remaining < 0: + raise AccessError('SSH configuration exceeds supported inspection limits; manual reconciliation required') + for line in content.decode().splitlines(): + line = line.lstrip() + if not line or line.startswith('#'): + continue + if line.startswith('"'): + keyword, separator, value = line[1:].partition('"') + if not separator: + raise AccessError('Unsupported SSH keyword quoting requires manual reconciliation') + else: + parsed = re.match(r'([^\s=]+)(.*)', line) + if not parsed: + raise AccessError('Unsupported SSH configuration syntax requires manual reconciliation') + keyword, value = parsed.groups() + value = value.lstrip() + if value.startswith('='): + value = value[1:] + if keyword.lower() == 'match': + raise AccessError('SSH Match directives require manual reconciliation before initialization') + if keyword.lower() != 'include': + continue + for pattern in include_arguments(value): + for included in expand(pattern): + inspect(included, ancestors | {identity}) + + try: + inspect(self.path('/etc/ssh/sshd_config'), set()) + except (OSError, UnicodeError) as exc: + raise AccessError('Cannot inspect SSH configuration and includes; manual reconciliation required') from exc + + def bootstrap_preflight(self): + for activity in ('is-active', 'is-enabled'): + if self.command(['systemctl', activity, '--quiet', 'ssh.socket'], False).returncode == 0: + raise AccessError('SSH socket activation requires manual handover; no access changes made') + if self.command(['systemctl', 'is-active', '--quiet', 'ssh.service'], False).returncode: + raise AccessError('An active ssh.service is required') + mode = self.command(['systemctl', 'show', 'ssh.service', '-p', 'KillMode', '--value']).stdout.strip() + if mode != 'process': + raise AccessError('ssh.service must preserve established sessions with KillMode=process') + arguments = self.service_arguments() + managed = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf') + if managed.exists(): + commands = [line.removeprefix('ExecStart=') for line in managed.read_text().splitlines() + if line.startswith('ExecStart=') and line != 'ExecStart='] + if len(commands) != 1 or arguments != shlex.split(commands[0]): + raise AccessError('Active SSH service configuration differs from its managed invocation') + elif arguments != ['/usr/sbin/sshd', '-D']: + raise AccessError('Custom SSH service invocation requires manual access reconciliation') + defaults = self.path('/etc/default/ssh') + if defaults.exists(): + for line in defaults.read_text().splitlines(): + if re.match(r'^\s*SSHD_OPTS\s*=', line): + value = line.split('=', 1)[1].strip().strip('\"\'') + if value: + raise AccessError('Custom SSHD_OPTS require manual access reconciliation') + self.unconditional_configuration() + self.command(['/usr/sbin/sshd', '-t']) + self.live_arguments() + + def preflight(self): + self.bootstrap_preflight() + self.command(['visudo', '-c']) + self.command(['runuser', '-u', 'adam', '--', 'sudo', '-n', 'true']) + + def service_arguments(self): + output = self.command(['systemctl', 'show', 'ssh.service', '-p', 'ExecStart', '--value']).stdout + matches = re.findall(r'argv\[\]=([^;]+)', output) + if len(matches) != 1: + raise AccessError('Unable to verify the SSH service invocation') + arguments = shlex.split(matches[0]) + # Debian's stock unit has this optional variable. Its actual expansion is + # independently checked against /proc below, so a nonempty value fails. + if arguments == ['/usr/sbin/sshd', '-D', '$SSHD_OPTS']: + arguments.pop() + if not arguments or arguments[0] != '/usr/sbin/sshd': + raise AccessError('SSH service does not invoke the supported daemon') + return arguments + + def live_arguments(self): + expected = self.service_arguments() + pid = self.command(['systemctl', 'show', 'ssh.service', '-p', 'MainPID', '--value']).stdout.strip() + if not pid.isdecimal() or int(pid) <= 1: + raise AccessError('SSH service has no verifiable active listener process') + process = self.path('/proc/' + pid) + try: + if os.readlink(process / 'exe') != '/usr/sbin/sshd': + raise AccessError('Active SSH process does not run the supported daemon') + parts = [part for part in (process / 'cmdline').read_bytes().decode().split('\0') if part] + except (OSError, UnicodeError): + raise AccessError('Unable to inspect the active SSH listener invocation') from None + if len(parts) == 1 and parts[0].startswith('sshd: ') and ' [listener]' in parts[0]: + actual = shlex.split(parts[0][6:].split(' [listener]', 1)[0]) + else: + actual = parts + if actual != expected: + raise AccessError('Running SSH listener differs from the active service invocation') + return actual, pid + + def verify_active(self, ports, final, address): + args, pid = self.live_arguments() + self.verify_policy([arg for arg in args if arg != '-D'], ports, final, address) + output = self.command(['ss', '-H', '-ltnp']).stdout + actual_ports = set() + for line in output.splitlines(): + if '("sshd",' not in line: + continue + if set(re.findall(r'pid=(\d+)', line)) != {pid}: + raise AccessError('An additional SSH listener exists outside the managed service') + fields = line.split() + if len(fields) < 5 or not fields[3].rsplit(':', 1)[-1].isdecimal(): + raise AccessError('Unable to verify the active SSH listening sockets') + actual_ports.add(int(fields[3].rsplit(':', 1)[-1])) + if actual_ports != set(ports): + raise AccessError('Active SSH listening ports differ from the declared policy') + + def policy(self, ports, final, address): + args = ['/usr/sbin/sshd'] + for port in sorted(set(ports)): + args += ['-p', str(port)] + if final: + for value in ('PermitRootLogin=no', 'PasswordAuthentication=no', + 'KbdInteractiveAuthentication=no', 'PubkeyAuthentication=yes', + 'AuthenticationMethods=publickey', 'AllowUsers=adam', + 'X11Forwarding=no', 'PermitEmptyPasswords=no'): + args += ['-o', value] + self.verify_policy(args, ports, final, address) + return args + + def verify_policy(self, args, ports, final, address): + self.unconditional_configuration() + self.command(args + ['-t']) + users = {'adam', 'root'} + if final: + users.update(account.pw_name for account in pwd.getpwall()) + for user in sorted(users): + output = self.command(args + ['-T', '-C', f'user={user},host=localhost,addr={address}']).stdout + values = {} + for line in output.splitlines(): + key, _, value = line.partition(' ') + values.setdefault(key, []).append(value) + if set(values.get('port', [])) != {str(p) for p in ports}: + raise AccessError('Effective SSH ports do not match the handover policy') + if final and any(values.get(k) != [v] for k, v in { + 'permitrootlogin': 'no', 'passwordauthentication': 'no', + 'kbdinteractiveauthentication': 'no', 'pubkeyauthentication': 'yes', + 'authenticationmethods': 'publickey', 'allowusers': 'adam', + 'x11forwarding': 'no', 'permitemptypasswords': 'no'}.items()): + raise AccessError('Effective SSH authentication policy is inconsistent') + + def current_final(self, address): + try: + self.verify_active([2222], True, address) + return True + except AccessError: + return False + + def install_policy(self, ports, final, address): + args = self.policy(ports, final, address) + path = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf') + atomic(path, ('[Service]\nExecStart=\nExecStart=' + ' '.join(args + ['-D']) + '\n').encode(), 0o644) + self.command(['systemctl', 'daemon-reload']) + actual = self.command(['systemctl', 'show', 'ssh.service', '-p', 'ExecStart', '--value']).stdout + match = re.search(r'argv\[\]=([^;]+)', actual) + if not match or shlex.split(match[1]) != args + ['-D']: + raise AccessError('Another service override prevents the managed SSH policy') + self.command(['systemctl', 'restart', 'ssh.service']) + self.verify_active(ports, final, address) + + def firewall_needed(self): + result = self.command(['ufw', 'status'], False) + if result.returncode or 'Status: active' not in result.stdout: + return False + if re.search(r'^2222/tcp(?:\s|\(v6\)).*ALLOW', result.stdout, re.MULTILINE): + return False + return True + + def open_firewall(self): + # Existing allows remain owned by their administrator. Only our commented rule is removed. + self.command(['ufw', 'allow', '2222/tcp', 'comment', 'apex-init-access']) + + def close_firewall(self): + self.command(['ufw', '--force', 'delete', 'allow', '2222/tcp', 'comment', 'apex-init-access']) + + def arm(self): + bundle = Path(__file__).resolve().parents[2] + if not re.fullmatch(r'/usr/local/lib/apex-provisioner/[a-f0-9]{64}', str(bundle)): + raise AccessError('Access runner must execute from the installed root-owned bundle') + service = ('[Unit]\nDescription=APEX pending access rollback\nAfter=local-fs.target\n' + '[Service]\nType=oneshot\nWorkingDirectory=' + str(bundle) + '\n' + 'ExecStart=/usr/bin/python3 -m engine.provisioning.access tick\n') + timer = ('[Unit]\nDescription=APEX access rollback deadline and boot recovery\n' + '[Timer]\nOnBootSec=5s\nOnUnitActiveSec=10s\nAccuracySec=1s\n' + '[Install]\nWantedBy=timers.target\n') + atomic(self.path('/etc/systemd/system/apex-access-rollback.service'), service.encode(), 0o644) + atomic(self.path('/etc/systemd/system/apex-access-rollback.timer'), timer.encode(), 0o644) + self.command(['systemctl', 'daemon-reload']) + self.command(['systemctl', 'enable', '--now', 'apex-access-rollback.timer']) + + def disarm(self): + self.command(['systemctl', 'disable', '--now', 'apex-access-rollback.timer']) + + def restore(self, state): + path = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf') + if state['previous'] is None: + path.unlink(missing_ok=True) + else: + atomic(path, base64.b64decode(state['previous']), 0o644) + self.command(['/usr/sbin/sshd', '-t']) + self.command(['systemctl', 'daemon-reload']) + self.command(['systemctl', 'restart', 'ssh.service']) + if state.get('firewall_added'): + self.close_firewall() + + +def atomic(path, data, mode=0o600): + path = Path(path) + for parent in [path, *path.parents]: + if parent.is_symlink(): + raise AccessError('Refusing a symlink in a managed access path') + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix='.apex-access-', dir=path.parent) + try: + with os.fdopen(fd, 'wb') as stream: + os.fchmod(stream.fileno(), mode) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + + +class Transition: + def __init__(self, host=None, now=time.monotonic, security=None): + from .security import Security + self.host = host or Host() + self.security = security if security is not None else Security(self.host) + self.now = now + self.directory = self.host.path('/var/lib/apex-init/access') + + def load(self): + path = self.directory / 'state.json' + return json.loads(path.read_text()) if path.exists() else None + + def save(self, state): + atomic(self.directory / 'state.json', json.dumps(state, sort_keys=True).encode()) + + def execute(self, operation, token=None, original_port=None, client_address=None): + # Rollback must never wait for a stalled enrollment stage holding init.lock. + if operation in ('tick', 'rollback'): + return self._execute(operation, token, original_port, client_address) + from .state import ProvisionError, exclusive_lock + try: + with exclusive_lock(self.host.path('/var/lib/apex-init/init.lock')): + return self._execute(operation, token, original_port, client_address) + except ProvisionError as error: + raise AccessError(str(error)) from None + + def _execute(self, operation, token=None, original_port=None, client_address=None): + if token is not None and not re.fullmatch(r'[a-f0-9]{32}', token): + raise AccessError('Invalid access transition token') + for path in [self.directory, *self.directory.parents]: + if path.is_symlink(): + raise AccessError('Invalid access state directory') + self.directory.mkdir(parents=True, exist_ok=True, mode=0o700) + if self.directory.stat().st_uid != os.geteuid(): + raise AccessError('Access state directory is not owned by the provisioner user') + os.chmod(self.directory, 0o700) + descriptor = os.open(self.directory / 'lock', os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, 'w') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + state = self.load() + if state and state['status'] == 'recovery_pending': + self._rollback(state) + if operation != 'tick': + raise AccessError('Pending recovery completed; retry initialization') + return {'status': 'rolled_back'} + pending = state and state['status'] in ('transition', 'final') + if pending and (self.now() >= state['deadline'] or self.host.boot_id() != state['boot_id']): + self._rollback(state) + if operation != 'tick': + raise AccessError('Uncommitted access transition expired or rebooted; previous access restored') + return {'status': 'rolled_back'} + if operation == 'tick': + return {'status': state['status'] if state else 'absent'} + if operation == 'begin': + if pending: + raise AccessError('A pending access transition exists; wait for rollback before retrying') + if not token or not original_port or not 1 <= original_port <= 65535: + raise AccessError('Access transition requires an original port and token') + try: + ipaddress.ip_address(client_address) + except ValueError: + raise AccessError('Invalid SSH client address') from None + self.host.preflight() + path = self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf') + preserve_policy = False + if state and state['status'] == 'committed': + if state['fingerprint'] != self.host.fingerprint(): + raise AccessError('Committed access configuration changed; reconcile before retrying') + self.host.verify_active([2222], True, client_address) + if state.get('security'): + self.security.verify(state['security']) + return {'status': 'committed'} + preserve_policy = True + if original_port == 2222 and not path.exists() and self.host.current_final(client_address): + preserve_policy = True + security_state = self.security.snapshot(self.security.preflight(), original_port, client_address) + self.host.arm() + state = {'status': 'transition', 'token': token, 'deadline': self.now() + 300, + 'boot_id': self.host.boot_id(), 'client_address': client_address, + 'original_port': original_port, 'firewall_added': False, + 'preserve_policy': preserve_policy, 'security': security_state, + 'previous': base64.b64encode(path.read_bytes()).decode() if path.exists() else None} + self.save(state) + try: + self.security.activate(state['security'], lambda value: self._save_security(state, value)) + if not preserve_policy: + self.host.install_policy([original_port, 2222], False, client_address) + state['fingerprint'] = self.host.fingerprint() + self.save(state) + except Exception: + self._rollback(state) + raise + return {'status': 'transition'} + if not state or (not pending and state['status'] != 'existing') or token != state['token']: + raise AccessError('No matching pending access transition') + if state['status'] == 'existing': + if operation == 'rollback': + state['status'] = 'rolled_back' + self.save(state) + return {'status': 'rolled_back'} + raise AccessError('Legacy access verification lacks security activation; retry initialization') + if operation == 'rollback': + self._rollback(state) + return {'status': 'rolled_back'} + if state.get('fingerprint') != self.host.fingerprint(): + raise AccessError('External SSH configuration changed; finalization stopped, rollback remains armed') + if operation == 'finalize': + if state['status'] != 'transition': + raise AccessError('Access transition is not ready for final policy') + try: + if not state.get('preserve_policy'): + self.host.install_policy([2222], True, state['client_address']) + self.security.finalize(state['security'], lambda value: self._save_security(state, value)) + state['status'] = 'final' + state['fingerprint'] = self.host.fingerprint() + self.save(state) + except Exception: + self._rollback(state) + raise + return {'status': 'final'} + if operation == 'commit' and state['status'] == 'final': + self.host.verify_active([2222], True, state['client_address']) + self.security.verify(state['security']) + state['status'] = 'committed' + self.save(state) # Timer observes this before it can restore anything. + self.host.disarm() + return {'status': 'committed'} + raise AccessError('Invalid access transition operation') + + def _save_security(self, state, value): + state['security'] = value + self.save(state) + + def _rollback(self, state): + errors = [] + if state.get('security'): + try: + self.security.restore(state['security'], lambda value: self._save_security(state, value)) + except Exception: + errors.append('security') + if not state.get('ssh_restored'): + try: + self.host.restore(state) + state['ssh_restored'] = True + except Exception: + errors.append('ssh') + if errors: + state['status'] = 'recovery_pending' + state['recovery_errors'] = errors + self.save(state) + raise AccessError('Recovery remains pending for ' + ', '.join(errors) + + '; watchdog remains armed; inspect the root-owned access record') + state['status'] = 'rolled_back' + state.pop('recovery_errors', None) + self.save(state) + self.host.disarm() + + +def main(argv=None): + from .security import SecurityError + parser = argparse.ArgumentParser() + parser.add_argument('operation', choices=['preflight', 'begin', 'finalize', 'commit', 'rollback', 'tick']) + parser.add_argument('--token') + parser.add_argument('--original-port', type=int) + parser.add_argument('--client-address') + args = parser.parse_args(argv) + if os.geteuid() != 0: + parser.error('Access handover requires root') + try: + if args.operation == 'preflight': + Host().bootstrap_preflight() + result = {'status': 'prerequisites-verified'} + else: + result = Transition().execute(args.operation, args.token, args.original_port, args.client_address) + print(json.dumps(result)) + return 0 + except (AccessError, SecurityError) as exc: + print(str(exc), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/engine/provisioning/baseline.py b/engine/provisioning/baseline.py new file mode 100644 index 0000000..27808d9 --- /dev/null +++ b/engine/provisioning/baseline.py @@ -0,0 +1,412 @@ +"""Shared Debian baseline. Callers own the exclusive initialization lock.""" +from __future__ import annotations + +from contextlib import contextmanager +import fcntl +from functools import wraps +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import urllib.request + + +class BaselineError(RuntimeError): + """A required baseline postcondition could not be established.""" + + +def expected_errors(operation): + @wraps(operation) + def guarded(*args, **kwargs): + try: + return operation(*args, **kwargs) + except (OSError, ValueError) as exc: + raise BaselineError("Baseline filesystem or saved state is invalid; inspect the target before retrying.") from exc + return guarded + + +BASE_PACKAGES = ("curl", "wget", "git", "rsyslog", "ufw") +ADMIN_PACKAGES = ( + "tcpdump", "socat", "rsync", "bind9-dnsutils", "iproute2", "net-tools", + "traceroute", "ca-certificates", "jq", "sed", "gawk", "tar", "gzip", + "bzip2", "xz-utils", "unzip", "htop", "iotop", "strace", "lsof", + "sysstat", "parted", "fdisk", "pciutils", "usbutils", "smartmontools", + "tmux", "vim", "less", "tree", "bash-completion", "python3", + "python3-minimal", "python3-yaml", "sudo", "cron", "iptables", "ipset", + "debian-archive-keyring", "kmod", +) +DOCKER_PACKAGES = ("docker-ce", "docker-ce-cli", "containerd.io", + "docker-buildx-plugin", "docker-compose-plugin") +SECURITY_PACKAGES = ("crowdsec", "crowdsec-firewall-bouncer-iptables") +DOCKER_CONFLICTS = ("docker.io", "docker-doc", "docker-compose", "docker-buildx", + "podman-docker", "containerd", "runc") +DOCKER_CONFIG = {"data-root": "/srv/docker", "log-driver": "json-file", + "log-opts": {"max-size": "20m", "max-file": "3"}} +HELPER_REVISION = "020a8699f95592561f254d8d4ad1bb40d401dfc7" +HELPER_URL = f"https://raw.githubusercontent.com/chaifeng/ufw-docker/{HELPER_REVISION}/ufw-docker" +HELPER_SHA256 = "643e56b080567c567b4aa28650196849b2a2da5dd0473fd3e5216b0886035ab0" +POLICY = Path("/usr/sbin/policy-rc.d") +POLICY_STATE = Path("/usr/sbin/.apex-policy-rc.d") +SECURITY_STATE = Path("/var/lib/apex/baseline-security.json") +SECURITY_SERVICES = {"ufw": "ufw", "crowdsec": "crowdsec", + "crowdsec-firewall-bouncer-iptables": "crowdsec-firewall-bouncer"} +SECURITY_CONDITION = "[Unit]\nConditionPathExists=/dev/null/apex-security-start-allowed\n" +APT = ["apt-get", "-o", "DPkg::Lock::Timeout=60", "-o", + "Acquire::Retries=2", "-o", "Acquire::http::Timeout=30", "-o", + "Acquire::https::Timeout=30"] + + +def run(args, *, check=True, timeout=300): + try: + result = subprocess.run(args, text=True, capture_output=True, timeout=timeout, + env={**os.environ, "LC_ALL": "C", "DEBIAN_FRONTEND": "noninteractive"}) + except (OSError, subprocess.TimeoutExpired) as exc: + raise BaselineError(f"Cannot complete {args[0]}: {exc}") from exc + if check and result.returncode: + raise BaselineError(f"{args[0]} failed with exit status {result.returncode}; inspect target system logs.") + return result + + +def preflight(*, strict=True): + if os.geteuid() != 0: + raise BaselineError("Baseline requires root.") + if not strict: + if not Path("/etc/debian_version").is_file(): + raise BaselineError("Base policy requires a Debian-based system.") + return + if not Path("/etc/os-release").is_file(): + raise BaselineError("Supported target is Debian 13 amd64 only.") + release = dict(line.split("=", 1) for line in Path("/etc/os-release").read_text().splitlines() + if "=" in line and not line.startswith("#")) + if (release.get("ID", "").strip('"') != "debian" or + release.get("VERSION_ID", "").strip('"') != "13" or + run(["dpkg", "--print-architecture"]).stdout.strip() != "amd64"): + raise BaselineError("Supported target is Debian 13 amd64 only.") + + +def package_versions(): + result = run(["dpkg-query", "-W", "-f=${binary:Package}\t${Status}\t${Version}\n"]) + installed = {} + for line in result.stdout.splitlines(): + fields = line.split("\t") + if len(fields) == 3 and fields[1].split()[1:] == ["ok", "installed"]: + installed[fields[0].removesuffix(":amd64")] = fields[2] + return installed + + +def write_managed(path, content, mode=0o644): + path = Path(path) + for parent in (path, *path.parents): + if parent.is_symlink(): + raise BaselineError(f"Refusing managed symlink: {parent}") + payload = content.encode() if isinstance(content, str) else content + if path.exists() and path.read_bytes() == payload: + if path.stat().st_mode & 0o777 != mode: + path.chmod(mode) + return False + path.parent.mkdir(parents=True, exist_ok=True) + fd, temporary = tempfile.mkstemp(prefix=".apex-", dir=path.parent) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary, mode) + os.replace(temporary, path) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + return True + + +def restore_service_policy(): + if not POLICY_STATE.exists(): + return + backup = POLICY_STATE / "original" + if (POLICY_STATE.is_symlink() or POLICY_STATE.stat().st_uid != os.geteuid()): + raise BaselineError("Refusing untrusted service policy recovery state.") + if (POLICY_STATE / "ready").exists() and POLICY.exists(): + original_matches = (backup.exists() and not POLICY.is_symlink() and + POLICY.read_bytes() == backup.read_bytes()) + symlink_matches = (backup.is_symlink() and POLICY.is_symlink() and + os.readlink(backup) == os.readlink(POLICY)) + guard_matches = not POLICY.is_symlink() and POLICY.read_bytes() == b"#!/bin/sh\nexit 101\n" + if not (original_matches or symlink_matches or guard_matches): + raise BaselineError("Service policy changed outside APEX; recovery requires explicit reconciliation.") + if backup.exists() or backup.is_symlink(): + # Copy first: an interrupted restoration must retain its recovery copy. + temporary = POLICY_STATE / "restoring" + if temporary.exists() or temporary.is_symlink(): + temporary.unlink() + shutil.copy2(backup, temporary, follow_symlinks=False) + os.replace(temporary, POLICY) + elif (POLICY_STATE / "absent").exists(): + POLICY.unlink(missing_ok=True) + elif (POLICY_STATE / "ready").exists(): + raise BaselineError("Service policy recovery is incomplete; inspect /usr/sbin/.apex-policy-rc.d.") + shutil.rmtree(POLICY_STATE) + + +@contextmanager +def service_start_guard(): + lock_path = Path("/run/lock/apex-package-policy.lock") + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise BaselineError("Another APEX package operation is running.") from None + with _service_start_guard(): + yield + + +@contextmanager +def _service_start_guard(): + restore_service_policy() + POLICY_STATE.mkdir(mode=0o700) + try: + if POLICY.exists() or POLICY.is_symlink(): + shutil.copy2(POLICY, POLICY_STATE / "original", follow_symlinks=False) + else: + (POLICY_STATE / "absent").touch() + (POLICY_STATE / "ready").touch() + # Replace the path itself, preserving any original symlink in the backup. + temporary = POLICY_STATE / "guard" + temporary.write_text("#!/bin/sh\nexit 101\n") + temporary.chmod(0o755) + os.replace(temporary, POLICY) + yield + finally: + restore_service_policy() + + +@contextmanager +def security_service_guard(packages): + """Guard new security units without occupying package-owned unit paths.""" + record = json.loads(SECURITY_STATE.read_text()) if SECURITY_STATE.exists() else {"version": 2, "services": {}} + if not isinstance(record, dict) or record.get("version") != 2: + raise BaselineError("Legacy security service guard state requires explicit reconciliation.") + managed = record.get("services") + if not isinstance(managed, dict) or any(SECURITY_SERVICES.get(key) != value for key, value in managed.items()): + raise BaselineError("Invalid baseline security service recovery state.") + installed = package_versions() + for package, service in SECURITY_SERVICES.items(): + if package in packages and package not in installed and package not in managed: + if run(["systemctl", "is-enabled", service], check=False).stdout.strip() == "masked": + continue + managed[package] = service + write_managed(SECURITY_STATE, json.dumps({"version": 2, "services": managed}), 0o600) + for service in managed.values(): + dropin = Path(f"/etc/systemd/system/{service}.service.d/99-apex-bootstrap-guard.conf") + if dropin.exists() and dropin.read_text() != SECURITY_CONDITION: + raise BaselineError("Security service guard changed outside APEX; reconcile it explicitly.") + # /dev/null cannot have children: this condition stays false after reboot. + write_managed(dropin, SECURITY_CONDITION) + if managed: + run(["systemctl", "daemon-reload"]) + try: + yield + finally: + installed = package_versions() + for package, service in managed.items(): + unit_exists = any(Path(f"{directory}/{service}.service").exists() for directory in + ("/etc/systemd/system", "/usr/lib/systemd/system", "/lib/systemd/system")) + run(["systemctl", "disable", service], check=package in installed or unit_exists) + dropin = Path(f"/etc/systemd/system/{service}.service.d/99-apex-bootstrap-guard.conf") + if dropin.exists(): + if dropin.is_symlink() or dropin.read_text() != SECURITY_CONDITION: + raise BaselineError("Security service guard changed outside APEX; reconcile it explicitly.") + dropin.unlink() + if managed: + run(["systemctl", "daemon-reload"]) + SECURITY_STATE.unlink(missing_ok=True) + + +def install_missing(packages): + installed = package_versions() + missing = [package for package in packages if package not in installed] + if not missing: + return [] + run([*APT, "update", "--error-on=any"]) + command = [*APT, "install", "-y", "--no-install-recommends", "--no-upgrade", "--no-remove", *missing] + plan = run([*command, "--simulate"]).stdout + if any(line.startswith("Remv ") or re.match(r"^Inst \S+ \[", line) + for line in plan.splitlines()): + raise BaselineError("Missing packages require an upgrade/removal; reconcile packages explicitly before retrying.") + # Package writes must settle even after coordinator loss; stages gate later work. + run(command, timeout=None) + remaining = set(missing) - package_versions().keys() + if remaining: + raise BaselineError("Required packages not installed: " + ", ".join(sorted(remaining))) + return missing + + +def base_policy(): + return {"packages": list(BASE_PACKAGES), "files": { + "/etc/apt/apt.conf.d/99-apex": 'APT::Keep-Downloaded-Packages "0";\n', + "/etc/systemd/journald.conf.d/99-apex.conf": "[Journal]\nSystemMaxUse=200M\n", + }, "helper": {"url": HELPER_URL, "sha256": HELPER_SHA256}} + + +def configure_common(): + policy = base_policy() + for legacy in ("/etc/apt/apt.conf.d/01clean", "/etc/systemd/journald.conf.d/maxuse.conf"): + path = Path(legacy) + if path.exists() or path.is_symlink(): + path.unlink() + for path, content in policy["files"].items(): + changed = write_managed(path, content) + if changed and path.endswith("99-apex.conf"): + run(["systemctl", "restart", "systemd-journald"]) + target = Path("/usr/local/bin/ufw-docker") + if (not target.is_file() or hashlib.sha256(target.read_bytes()).hexdigest() != HELPER_SHA256): + try: + with urllib.request.urlopen(HELPER_URL, timeout=30) as response: + data = response.read() + except OSError as exc: + raise BaselineError("Unable to download pinned ufw-docker helper.") from exc + if hashlib.sha256(data).hexdigest() != HELPER_SHA256: + raise BaselineError("Pinned ufw-docker checksum mismatch.") + write_managed(target, data, 0o755) + elif target.is_symlink(): + raise BaselineError("Refusing a symlinked ufw-docker helper.") + else: + target.chmod(0o755) + + +def docker_preflight(installed): + conflicts = sorted(set(DOCKER_CONFLICTS) & installed.keys()) + if conflicts: + raise BaselineError("Conflicting Docker packages require explicit reconciliation: " + ", ".join(conflicts)) + path = Path("/etc/docker/daemon.json") + try: + config = json.loads(path.read_text()) if path.exists() else {} + except (OSError, ValueError) as exc: + raise BaselineError("Cannot validate existing Docker daemon configuration.") from exc + if not isinstance(config, dict): + raise BaselineError("Docker daemon configuration must be an object.") + for name, value in DOCKER_CONFIG.items(): + if name in config and config[name] != value: + raise BaselineError(f"Conflicting Docker {name}; reconcile explicitly before initialization.") + old_data = Path("/var/lib/docker") + if old_data.exists() and any(old_data.iterdir()): + raise BaselineError("Existing /var/lib/docker data must be reconciled explicitly; no data migration is automatic.") + data = Path("/srv/docker") + if data.is_symlink() or data.parent.is_symlink(): + raise BaselineError("Refusing a redirected Docker data-root.") + if data.exists() and any(data.iterdir()) and config.get("data-root") != "/srv/docker": + raise BaselineError("Unmanaged /srv/docker data requires explicit reconciliation.") + if "docker-ce" in installed: + active = run(["systemctl", "is-active", "--quiet", "docker"], check=False).returncode == 0 + if active: + root = run(["docker", "info", "--format", "{{.DockerRootDir}}"] ).stdout.strip() + if root != "/srv/docker" or any(config.get(key) != value for key, value in DOCKER_CONFIG.items()): + raise BaselineError("Running Docker has incompatible data-root/log policy; reconcile it before retrying.") + return {**config, **DOCKER_CONFIG} + + +def configure_sources(): + write_managed("/etc/apt/sources.list.d/apex-zfs.sources", + "Types: deb\nURIs: https://deb.debian.org/debian\nSuites: trixie\n" + "Components: contrib\nArchitectures: amd64\n" + "Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg\n") + for name, key_url, uri, suite, component in ( + ("docker", "https://download.docker.com/linux/debian/gpg", + "https://download.docker.com/linux/debian", "trixie", "stable"), + ("crowdsec", "https://packagecloud.io/crowdsec/crowdsec/gpgkey", + "https://packagecloud.io/crowdsec/crowdsec/any", "any", "main"), + ): + key = Path(f"/etc/apt/keyrings/apex-{name}.asc") + if not key.exists(): + try: + with urllib.request.urlopen(key_url, timeout=30) as response: + data = response.read() + except OSError as exc: + raise BaselineError(f"Cannot download {name} repository signing key.") from exc + if not data.startswith(b"-----BEGIN PGP PUBLIC KEY BLOCK-----"): + raise BaselineError(f"Invalid {name} repository signing key.") + write_managed(key, data) + write_managed(f"/etc/apt/sources.list.d/apex-{name}.sources", + f"Types: deb\nURIs: {uri}\nSuites: {suite}\nComponents: {component}\n" + f"Architectures: amd64\nSigned-By: {key}\n") + + +def ensure_service(name): + if run(["systemctl", "is-enabled", "--quiet", name], check=False).returncode: + run(["systemctl", "enable", name]) + if run(["systemctl", "is-active", "--quiet", name], check=False).returncode: + run(["systemctl", "start", name]) + + +@expected_errors +def apply_base_policy(): + """The bounded configure/base policy: no Docker, storage or enrollment.""" + preflight(strict=False) + with service_start_guard(), security_service_guard(BASE_PACKAGES): + install_missing(BASE_PACKAGES) + configure_common() + ensure_service("rsyslog") + run(["/usr/local/bin/ufw-docker", "install"]) + return {"packages": {key: value for key, value in package_versions().items() if key in BASE_PACKAGES}} + + +@expected_errors +def apply_baseline(): + """Converge baseline and return verified package versions and capabilities.""" + preflight() + installed = package_versions() + docker_config = docker_preflight(installed) + kernel = run(["uname", "-r"]).stdout.strip() + if not re.fullmatch(r"[A-Za-z0-9.+_-]+", kernel): + raise BaselineError("Unsupported running kernel release.") + packages = (*BASE_PACKAGES, *ADMIN_PACKAGES, *DOCKER_PACKAGES, + *SECURITY_PACKAGES, "linux-headers-amd64", f"linux-headers-{kernel}", + "zfsutils-linux", "zfs-dkms") + with service_start_guard(), security_service_guard(packages): + install_missing((*BASE_PACKAGES, *ADMIN_PACKAGES)) + configure_sources() + install_missing(packages) + configure_common() + write_managed("/etc/docker/daemon.json", json.dumps(docker_config, indent=2) + "\n") + Path("/srv/docker").mkdir(parents=True, exist_ok=True) + ensure_service("containerd") + ensure_service("docker") + ensure_service("rsyslog") + ensure_service("cron") + root = run(["docker", "info", "--format", "{{.DockerRootDir}}"] ).stdout.strip() + if root != "/srv/docker": + raise BaselineError("Docker did not activate the required /srv/docker data-root.") + compose = run(["docker", "compose", "version", "--short"]).stdout.strip() + if not compose: + raise BaselineError("Docker Compose capability is unavailable.") + module = run(["modprobe", "zfs"], check=False) + if module.returncode: + raise BaselineError(f"ZFS unavailable for running kernel {kernel}; install matching headers/module or reboot into the supported kernel, then retry. No automatic reboot was performed.") + zfs = run(["zfs", "version"]).stdout.strip() + versions = package_versions() + if set(packages) - versions.keys(): + raise BaselineError("Baseline package verification failed.") + return {"packages": {name: versions[name] for name in packages}, + "capabilities": {"docker_data_root": root, "compose": compose, "zfs": zfs, + "kernel": kernel, "ufw_docker_sha256": HELPER_SHA256}, + "sources": {"docker": "docker.com/debian trixie stable", + "crowdsec": "packagecloud.io/crowdsec/crowdsec/any any main", + "zfs": "Debian trixie contrib"}} + + +if __name__ == "__main__": + import argparse + from .state import ProvisionError, exclusive_lock + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-policy", action="store_true") + args = parser.parse_args() + try: + preflight(strict=not args.base_policy) + with exclusive_lock("/var/lib/apex-init/init.lock"): + print(json.dumps(apply_base_policy() if args.base_policy else apply_baseline(), sort_keys=True)) + except (BaselineError, OSError, ProvisionError) as exc: + parser.exit(1, f"Baseline failed: {exc}\n") diff --git a/engine/provisioning/contract.py b/engine/provisioning/contract.py new file mode 100644 index 0000000..f41d6a1 --- /dev/null +++ b/engine/provisioning/contract.py @@ -0,0 +1,236 @@ +"""Pure validation for the supported APEX enrollment contract.""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import PurePosixPath +from typing import Any, Dict, Mapping, Optional, Tuple +from urllib.parse import urlsplit + + +class ContractError(ValueError): + """A safe diagnostic that never includes a supplied value.""" + + +@dataclass(frozen=True) +class Repository: + name: str + url: str + ref: Optional[str] + + +@dataclass(frozen=True) +class Tier: + number: int + mode: str + device: Optional[str] + pool_guid: Optional[str] + + +@dataclass(frozen=True) +class ManagedFile: + path: str + owner: str + mode: int + content: str = field(repr=False) + + +@dataclass(frozen=True) +class Enrollment: + hostname: str + repository: Repository + tiers: Tuple[Tier, ...] + authorized_keys: Tuple[str, ...] = field(repr=False) + files: Tuple[ManagedFile, ...] = field(repr=False) + + def identity(self) -> Dict[str, Any]: + return { + 'hostname': self.hostname, + 'repository': { + 'name': self.repository.name, + 'url': self.repository.url, + 'ref': self.repository.ref, + }, + 'tiers': [ + {'number': t.number, 'mode': t.mode, + 'device': t.device, 'pool_guid': t.pool_guid} + for t in self.tiers + ], + } + + +_ENV_KEYS = {'APEX_REPO_NAME', 'APEX_REPO_URL', 'APEX_REPO_REF'} | { + f'APEX_TIER{number}_{suffix}' + for number in (1, 2, 3) + for suffix in ('DEVICE', 'MODE', 'POOL_GUID') +} +_ASSIGNMENT = re.compile( + r'''\s*([A-Z][A-Z0-9_]*)\s*=\s*(?:"([^"\\]*)"|'([^'\\]*)'|([^\s'"\\()#]*))\s*''' +) +_FILE_POLICY = { + '/etc/apex/bootstrap.env': ('root:root', 0o600, False), + '/home/adam/.ssh/id_ed25519': ('adam:adam', 0o600, True), + '/home/adam/.ssh/known_hosts': ('adam:adam', 0o644, True), +} + + +def _require(condition: bool, diagnostic: str) -> None: + if not condition: + raise ContractError(diagnostic) + + +def _mapping(value: Any, required: set, optional: set, diagnostic: str) -> Mapping: + _require(isinstance(value, Mapping), diagnostic) + _require(set(value) >= required and set(value) <= required | optional, diagnostic) + return value + + +def _plain(value: Any, diagnostic: str, allow_empty: bool = False) -> str: + _require(type(value) is str, diagnostic) + _require(allow_empty or bool(value), diagnostic) + _require(not any(ord(c) < 32 or ord(c) == 127 for c in value), diagnostic) + return value + + +def parse_bootstrap_env(text: str) -> Dict[str, str]: + _require(type(text) is str, 'bootstrap.env must contain literal assignments') + result = {} + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith('#'): + continue + match = _ASSIGNMENT.fullmatch(line) + _require(match is not None, 'bootstrap.env must contain literal assignments') + key = match.group(1) + _require(key in _ENV_KEYS, 'bootstrap.env contains an unsupported assignment') + _require(key not in result, 'bootstrap.env contains a duplicate assignment') + value = next(part for part in match.groups()[1:] if part is not None) + _require(not any(c in value for c in '$`;|&<>'), + 'bootstrap.env contains unsupported shell syntax') + result[key] = _plain(value, 'bootstrap.env contains an invalid literal', True) + return result + + +def _repository(values: Mapping[str, str]) -> Repository: + name = values.get('APEX_REPO_NAME', '') + _require(bool(re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._-]*', name)) + and name not in ('.', '..'), 'repository name must be one safe directory name') + url = values.get('APEX_REPO_URL', '') + _require(bool(url) and not any(c.isspace() for c in url), 'repository URL is invalid') + if '://' in url: + try: + parsed = urlsplit(url) + port = parsed.port + except ValueError: + raise ContractError('repository URL is invalid') from None + _require(parsed.scheme in ('https', 'ssh') and bool(parsed.hostname) + and parsed.password is None and not parsed.query and not parsed.fragment + and bool(parsed.path.strip('/')) and (port is None or port > 0), + 'repository URL is invalid or contains credentials') + _require(parsed.scheme != 'https' or parsed.username is None, + 'HTTPS repository URL must not contain credentials') + else: + _require(bool(re.fullmatch( + r'[A-Za-z0-9._-]+@[A-Za-z0-9.-]+:[A-Za-z0-9_./-]+', url)), + 'repository URL must use HTTPS or SSH') + remote_path = url.split(':', 1)[1] + _require(not remote_path.startswith('-') and remote_path not in ('.', '..'), + 'repository URL is invalid') + ref = values.get('APEX_REPO_REF') or None + if ref is not None: + _require(not ref.startswith('-') and not any(c.isspace() for c in ref), + 'repository ref is invalid') + return Repository(name, url, ref) + + +def _tiers(values: Mapping[str, str]) -> Tuple[Tier, ...]: + result = [] + devices = set() + guids = set() + for number in (1, 2, 3): + prefix = f'APEX_TIER{number}_' + device = values.get(prefix + 'DEVICE') or None + mode = values.get(prefix + 'MODE') + guid = values.get(prefix + 'POOL_GUID') or None + _require(mode is not None or device is None, + 'a mapped tier device requires an explicit mode') + mode = mode or 'directory' + _require(mode in ('directory', 'create', 'import'), 'tier mode is invalid') + if mode == 'directory': + _require(device is None and guid is None, + 'directory tier must not select a device or pool') + else: + _require(device is not None and device.startswith('/dev/') + and not any(c.isspace() for c in device) + and '..' not in PurePosixPath(device).parts + and str(PurePosixPath(device)) == device, + 'block-backed tier requires a normalized device path') + _require(device not in devices, 'a device is selected by multiple tiers') + devices.add(device) + if mode == 'create': + _require(guid is None, 'create tier must not name an existing pool') + else: + _require(guid is not None and bool(re.fullmatch(r'[1-9][0-9]{0,19}', guid)) + and int(guid) < 2**64, 'import tier requires a valid pool GUID') + _require(guid not in guids, 'a pool is selected by multiple tiers') + guids.add(guid) + result.append(Tier(number, mode, device, guid)) + return tuple(result) + + +def normalize_user_data(document: Any) -> Enrollment: + doc = _mapping(document, + {'preserve_hostname', 'hostname', 'users', 'write_files', 'runcmd'}, + set(), 'unsupported cloud-config structure') + _require(doc['preserve_hostname'] is False, 'preserve_hostname must be false') + hostname = _plain(doc['hostname'], 'hostname is invalid') + labels = hostname.split('.') + _require(len(hostname) <= 253 and len(labels) >= 2 + and all(re.fullmatch(r'[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?', x) + for x in labels), 'hostname must be a valid FQDN') + users = doc['users'] + _require(type(users) is list and len(users) == 1, 'exactly one administrator is required') + user = _mapping(users[0], + {'name', 'sudo', 'groups', 'shell', 'ssh_authorized_keys'}, + set(), 'unsupported administrator configuration') + _require(user['name'] == 'adam' and user['shell'] == '/bin/bash', + 'administrator must be adam with Bash') + _require(type(user['sudo']) is str and ''.join(user['sudo'].split()) in + ('ALL=(ALL)NOPASSWD:ALL', 'ALL=(ALL:ALL)NOPASSWD:ALL'), + 'administrator requires passwordless sudo') + groups = user['groups'] + _require(type(groups) is list and len(groups) == 2 + and all(type(g) is str for g in groups) and set(groups) == {'sudo', 'docker'}, + 'administrator requires sudo and docker groups') + keys = user['ssh_authorized_keys'] + _require(type(keys) is list and len(keys) > 0, 'administrator requires public SSH keys') + authorized_keys = tuple(_plain(k, 'administrator SSH key is invalid') for k in keys) + _require(all(k.strip() for k in authorized_keys), 'administrator SSH key is empty') + commands = doc['runcmd'] + _require(commands in (['/usr/local/bin/apex-bootstrap'], + [['/usr/local/bin/apex-bootstrap']]), + 'only the standard APEX bootstrap command is supported') + writes = doc['write_files'] + _require(type(writes) is list, 'write_files must be a list') + files = [] + seen = set() + for write in writes: + item = _mapping(write, {'path', 'owner', 'permissions', 'content'}, {'defer'}, + 'unsupported write_files entry') + path = _plain(item['path'], 'managed file path is invalid') + _require(path in _FILE_POLICY and path not in seen, + 'managed file path is unsupported or repeated') + seen.add(path) + owner, mode, deferred = _FILE_POLICY[path] + permission = item['permissions'] + _require(type(permission) is str and bool(re.fullmatch(r'0?[0-7]{3}', permission)) + and int(permission, 8) == mode and item['owner'] == owner, + 'managed file ownership or permissions are invalid') + _require(item.get('defer', False) is deferred, 'managed file defer setting is invalid') + content = item['content'] + _require(type(content) is str and bool(content.strip()) and '\x00' not in content, + 'managed file content is invalid') + files.append(ManagedFile(path, owner, mode, content)) + bootstrap = next((f for f in files if f.path == '/etc/apex/bootstrap.env'), None) + _require(bootstrap is not None, 'bootstrap.env is required') + values = parse_bootstrap_env(bootstrap.content) + return Enrollment(hostname, _repository(values), _tiers(values), authorized_keys, tuple(files)) diff --git a/engine/provisioning/image.py b/engine/provisioning/image.py new file mode 100644 index 0000000..c863b66 --- /dev/null +++ b/engine/provisioning/image.py @@ -0,0 +1,124 @@ +"""Image-only installation and sanitization; never called by remote enrollment.""" +from __future__ import annotations + +import argparse +import io +import json +import os +from pathlib import Path +import shutil +import tarfile +import uuid + +from . import ENROLLMENT_INTERFACE +from .state import ProvisionError, atomic_write, exclusive_lock, safe_path +from .target import preflight, run + + +def install(): + from .transport import build_bundle + source = Path(__file__).resolve().parents[2] + data, digest, _ = build_bundle(source) + destination = safe_path(Path('/usr/local/lib/apex-provisioner') / digest) + destination.mkdir(mode=0o755, parents=True, exist_ok=False) + with tarfile.open(fileobj=io.BytesIO(data)) as archive: + archive.extractall(destination, filter='data') + revision = os.environ.get('APEX_PROVISION_REVISION', 'unversioned-modified') + metadata = {'bundle_digest': digest, 'revision': revision, 'path': str(destination), + 'baseline_revision': revision, 'enrollment_interface': ENROLLMENT_INTERFACE} + atomic_write('/etc/apex/provisioner.json', json.dumps(metadata) + '\n') + for executable, operation in (('apex-provision-image', ''), ('apex-bootstrap', 'cloud-enroll ')): + content = ('#!/bin/sh\nset -eu\ncd ' + str(destination) + '\n' + 'exec /usr/bin/python3 -m engine.provisioning.image ' + operation + '"$@"\n') + atomic_write('/usr/local/bin/' + executable, content, 0o755) + return metadata + + +def cloud_enroll(): + from .access import Transition + from .target import enroll + from .userdata import decode_user_data + metadata = json.loads(Path('/etc/apex/provisioner.json').read_text()) + if metadata.get('enrollment_interface') != ENROLLMENT_INTERFACE: + raise ProvisionError('image enrollment interface is missing or unsupported; rebuild the image') + instance = Path('/var/lib/cloud/instance').resolve(strict=True) + try: + instance.relative_to('/var/lib/cloud/instances') + except ValueError: + raise RuntimeError('cloud-init instance path is outside its managed directory') from None + path = safe_path(instance / 'user-data.txt') + if path.stat().st_uid != 0 or path.stat().st_size > 1024 * 1024: + raise RuntimeError('cloud-init user-data is not a supported root-owned file') + spec = decode_user_data(path.read_bytes()) + result = enroll(spec, metadata['revision'], metadata['bundle_digest']) + token = uuid.uuid4().hex + transition = Transition() + try: + current = transition.execute('begin', token, 22, '127.0.0.1') + run(['runuser', '-u', 'adam', '--', 'sudo', '-n', 'true']) + if current['status'] not in ('existing', 'committed'): + transition.execute('finalize', token) + if current['status'] != 'committed': + transition.execute('commit', token) + except Exception: + transition.execute('rollback', token) + raise + result['external_access'] = 'not-verified-cloud-init' + atomic_write('/var/lib/apex-init/enrollment.json', json.dumps(result, sort_keys=True) + '\n') + return result + + +def sanitize_files(root=Path('/')): + root = Path(root) + for home in ('root', 'home/debian'): + for name in ('.ssh/authorized_keys', '.ssh/authorized_keys2', '.bash_history'): + path = root / home / name + if path.exists() or path.is_symlink(): + path.unlink() + for path in (root / 'etc/ssh').glob('ssh_host_*'): + if path.is_file() or path.is_symlink(): + path.unlink() + atomic_write(root / 'etc/machine-id', '', 0o444) + dbus = root / 'var/lib/dbus/machine-id' + if dbus.exists() or dbus.is_symlink(): + dbus.unlink() + dbus.parent.mkdir(parents=True, exist_ok=True) + dbus.symlink_to('/etc/machine-id') + for name in ('var/lib/apex-init', 'tmp/engine', 'var/lib/cloud'): + path = root / name + if path.is_symlink(): + path.unlink() + elif path.exists(): + shutil.rmtree(path) + autologin = root / 'etc/systemd/system/serial-getty@ttyS0.service.d/autologin.conf' + autologin.unlink(missing_ok=True) + atomic_write(root / 'etc/systemd/system/ssh.service.d/10-generate-host-keys.conf', + '[Service]\nExecStartPre=\nExecStartPre=/usr/bin/ssh-keygen -A\n' + 'ExecStartPre=/usr/sbin/sshd -t\n', 0o644) + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('operation', choices=('install', 'baseline', 'install-runner', 'sanitize', 'cloud-enroll')) + args = parser.parse_args(argv) + preflight() + if args.operation == 'install': + print(json.dumps(install())) + elif args.operation == 'baseline': + from .baseline import apply_baseline + with exclusive_lock('/var/lib/apex-init/init.lock'): + result = apply_baseline() + atomic_write('/etc/apex/image-baseline.json', json.dumps(result, sort_keys=True) + '\n') + elif args.operation == 'install-runner': + run(['test', '-x', '/usr/local/bin/apex-bootstrap']) + elif args.operation == 'sanitize': + run(['usermod', '--password', '!', 'root']) + run(['usermod', '--password', '!', 'debian']) + sanitize_files() + else: + print(json.dumps(cloud_enroll(), sort_keys=True)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/engine/provisioning/repository.py b/engine/provisioning/repository.py new file mode 100644 index 0000000..fbeec54 --- /dev/null +++ b/engine/provisioning/repository.py @@ -0,0 +1,121 @@ +"""Pinned, non-destructive checkout enrollment as the node administrator.""" +from __future__ import annotations + +import os +from pathlib import Path +import pwd +import subprocess + +from engine.identity import read_env +from . import ENROLLMENT_INTERFACE +from .state import ProvisionError, safe_path + + +def as_adam(args, cwd=None): + environment = dict(os.environ, GIT_TERMINAL_PROMPT='0', + GIT_SSH_COMMAND='ssh -o BatchMode=yes -o StrictHostKeyChecking=yes', + LC_ALL='C') + result = subprocess.run(['runuser', '-u', 'adam', '--', *args], cwd=cwd, + env=environment, text=True, capture_output=True, timeout=300) + if result.returncode: + # Git/hook stderr may contain credential files or remote URL credentials. + raise ProvisionError('repository operation failed; check administrator Git access and the pinned checkout') + return result.stdout.strip() + + +def verify_checkout_setup(destination, account): + if as_adam(['git', 'config', '--get', 'core.hooksPath'], destination) != 'commons/githooks': + raise ProvisionError('checkout hook configuration is missing or incompatible') + home = safe_path(account.pw_dir) + bashrc = safe_path(home / '.bashrc') + entry = 'export PATH="' + str(destination / 'commons') + ':$PATH"' + if not bashrc.is_file() or bashrc.read_text().splitlines().count(entry) != 1: + raise ProvisionError('checkout PATH setup is missing or duplicated; reconcile init.sh before retrying') + vim = home / '.vimrc' + if not vim.is_symlink() or vim.resolve() != (destination / 'commons/etc/.vimrc').resolve(): + raise ProvisionError('checkout editor link is missing or incompatible') + + +def reconcile_repository(spec, state, save, srv=Path('/srv')): + repository = spec.repository + destination = safe_path(Path(srv) / repository.name) + staging = safe_path(Path(srv) / ('.apex-clone-' + repository.name)) + expected = {'name': repository.name, 'url': repository.url, 'ref': repository.ref} + account = pwd.getpwnam('adam') + record = state.setdefault('repository', {}) + if record and record.get('request') != expected: + raise ProvisionError('repository enrollment conflicts with its recorded request') + if not record: + if destination.exists() or staging.exists(): + raise ProvisionError('unrecorded repository path exists; explicit adoption is required') + record.update(request=expected, clone_intent=True) + save() + if not destination.exists(): + if staging.exists(): + # An interrupted clone may be incomplete: never erase or guess its state. + raise ProvisionError('an interrupted clone remains; inspect the staging checkout before retrying') + staging.mkdir(mode=0o755) + os.chown(staging, account.pw_uid, account.pw_gid) + as_adam(['git', 'clone', '--no-checkout', '--', repository.url, str(staging)]) + os.rename(staging, destination) + safe_path(destination / '.git') + origin = as_adam(['git', 'remote', 'get-url', 'origin'], destination) + if origin != repository.url: + raise ProvisionError('existing checkout has a different origin; it was preserved') + if 'commit' not in record: + reference = repository.ref or 'HEAD' + if repository.ref: + as_adam(['git', 'fetch', '--no-tags', 'origin', reference], destination) + reference = 'FETCH_HEAD' + commit = as_adam(['git', 'rev-parse', '--verify', reference + '^{commit}'], destination) + record['commit'] = commit + record['checkout_intent'] = True + save() + commit = record['commit'] + if record.get('checkout_intent'): + # --no-checkout index intentionally looks dirty before the first checkout. + if any(p.name != '.git' for p in destination.iterdir()): + head = as_adam(['git', 'rev-parse', 'HEAD'], destination) + dirty = as_adam(['git', 'status', '--porcelain'], destination) + if head != commit or dirty: + raise ProvisionError('interrupted checkout has unexpected contents; it was preserved') + else: + as_adam(['git', 'checkout', '--detach', commit], destination) + record.pop('checkout_intent') + save() + if as_adam(['git', 'rev-parse', 'HEAD'], destination) != commit: + raise ProvisionError('checkout revision changed; initialization will not reset it') + if as_adam(['git', 'status', '--porcelain'], destination): + raise ProvisionError('checkout contains local changes; initialization will not reset it') + node_env = safe_path(destination / 'node.env') + node_identity = read_env(str(node_env)) + if node_identity.get('APEX_NODE_FQDN') != spec.hostname: + raise ProvisionError('repository node.env does not declare the requested hostname') + if node_identity.get('APEX_ENROLLMENT_INTERFACE') != str(ENROLLMENT_INTERFACE): + raise ProvisionError('node checkout enrollment interface is missing or unsupported; update its init.sh and node.env together') + as_adam(['git', 'submodule', 'update', '--init', '--recursive'], destination) + commons = safe_path(destination / 'commons') + if not commons.is_dir(): + raise ProvisionError('node checkout does not contain its pinned commons submodule') + commons_commit = as_adam(['git', 'rev-parse', 'HEAD'], commons) + gitlink = as_adam(['git', 'ls-tree', 'HEAD', '--', 'commons'], destination).split() + if gitlink != ['160000', 'commit', commons_commit, 'commons']: + raise ProvisionError('commons is not the exact pinned Git submodule') + if Path(as_adam(['git', 'rev-parse', '--show-toplevel'], commons)).resolve() != commons.resolve(): + raise ProvisionError('commons does not have its own Git worktree') + if record.get('commons_commit', commons_commit) != commons_commit: + raise ProvisionError('pinned commons revision changed unexpectedly') + hook = safe_path(destination / 'init.sh') + if not hook.is_file(): + raise ProvisionError('node repository is missing its checkout initializer') + if not record.get('hook_complete'): + as_adam(['bash', './init.sh'], destination) + verify_checkout_setup(destination, account) + record['hook_complete'] = True + else: + verify_checkout_setup(destination, account) + if as_adam(['git', 'status', '--porcelain'], destination): + raise ProvisionError('checkout initializer changed tracked files; inspect before resuming') + record['commons_commit'] = commons_commit + save() + return {'path': str(destination), 'commit': commit, 'commons_commit': commons_commit} diff --git a/engine/provisioning/security.py b/engine/provisioning/security.py new file mode 100644 index 0000000..b3ecf22 --- /dev/null +++ b/engine/provisioning/security.py @@ -0,0 +1,475 @@ +"""Security activation owned by the protected access transaction.""" +from __future__ import annotations + +import base64 +import hashlib +import ipaddress +import json +import os +import re +import shlex +import uuid + +from .access import atomic +from .baseline import HELPER_SHA256 + + +SERVICES = ('ufw', 'crowdsec', 'crowdsec-firewall-bouncer') +FILES = ('/etc/default/ufw', '/etc/ufw/ufw.conf', '/etc/ufw/before.rules', + '/etc/ufw/before6.rules', '/etc/ufw/after.rules', '/etc/ufw/after6.rules', + '/etc/ufw/user.rules', '/etc/ufw/user6.rules') +HELPER = '/usr/local/bin/ufw-docker' +BLOCK = re.compile(r'^# BEGIN UFW AND DOCKER\n.*?^# END UFW AND DOCKER\n?', re.M | re.S) + + +def _helper_lines(ipv6): + prefix = 'ufw6' if ipv6 else 'ufw' + subnets = ('fd00::/8',) if ipv6 else ('10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16') + return ['# BEGIN UFW AND DOCKER', '*filter', ':' + prefix + '-user-forward - [0:0]', + ':' + prefix + '-docker-logging-deny - [0:0]', ':DOCKER-USER - [0:0]', + '-A DOCKER-USER -j ' + prefix + '-user-forward', + '-A DOCKER-USER -m conntrack --ctstate RELATED,ESTABLISHED -j RETURN', + '-A DOCKER-USER -m conntrack --ctstate INVALID -j DROP', + '-A DOCKER-USER -i docker0 -o docker0 -j ACCEPT', + *['-A DOCKER-USER -j RETURN -s ' + subnet for subnet in subnets], + *['-A DOCKER-USER -j ' + prefix + '-docker-logging-deny -m conntrack --ctstate NEW -d ' + subnet + for subnet in subnets], + '-A DOCKER-USER -j RETURN', + '-A ' + prefix + '-docker-logging-deny -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW DOCKER BLOCK] "', + '-A ' + prefix + '-docker-logging-deny -j DROP', 'COMMIT', '# END UFW AND DOCKER'] + + +def _runtime_rule(line): + tokens = shlex.split(line) + if len(tokens) % 2: + raise SecurityError('Unsupported runtime firewall rule') + pairs = [] + for key, value in zip(tokens[::2], tokens[1::2]): + if key in ('-s', '-d'): + value = str(ipaddress.ip_network(value, strict=False)) + elif key == '--ctstate': + value = ','.join(sorted(value.split(','))) + pairs.append((key, value)) + return sorted(pairs) + + +class SecurityError(RuntimeError): + pass + + +class Security: + def __init__(self, host): + self.host = host + + def preflight(self): + services = {name: self._service(name) for name in SERVICES} + for name, state in services.items(): + if (state.get('LoadState') != 'loaded' or state.get('UnitFileState') not in ('enabled', 'disabled') + or state.get('ActiveState') not in ('active', 'inactive') + or state.get('DropInPaths') or state.get('FragmentPath') not in + ('/usr/lib/systemd/system/' + name + '.service', '/lib/systemd/system/' + name + '.service')): + raise SecurityError('Unsupported or masked security service: ' + name) + if self._digest(self._path(HELPER).read_bytes()) != HELPER_SHA256: + raise SecurityError('The ufw-docker helper does not match the pinned baseline') + files = self._files() + active = self._active() + pristine = False + if not active: + output = self.host.command(['dpkg-query', '-W', '-f=${Conffiles}', 'ufw']).stdout + conffiles = dict(re.findall(r'^\s*(/\S+)\s+([a-f0-9]{32})(?:\s|$)', output, re.M)) + pristine = (hashlib.md5(base64.b64decode(files['/etc/default/ufw']['data'])).hexdigest() + == conffiles.get('/etc/default/ufw')) + manifest = dict((name, digest) for digest, name in re.findall( + r'^([a-f0-9]{32})\s+(\S+)\s*$', self._text('/var/lib/dpkg/info/ufw.md5sums'), re.M)) + # Debian's postinst copies/ucf-installs these files; they are not dpkg conffiles. + for name in FILES[1:]: + filename = name.rsplit('/', 1)[1] + template = '/usr/share/ufw/' + ('' if filename == 'ufw.conf' else 'iptables/') + filename + data = self._path(template).read_bytes() + pristine = pristine and (hashlib.md5(data).hexdigest() == manifest.get(template.lstrip('/')) + and data == base64.b64decode(files[name]['data'])) + if not pristine: + raise SecurityError('Inactive UFW is not provably pristine; reconcile its existing policy first') + texts = [self._text('/etc/ufw/' + name) for name in ('after.rules', 'after6.rules')] + present = any('UFW AND DOCKER' in text for text in texts) + if present: + self._check_helper_files() + else: + for executable in ('iptables', 'ip6tables'): + result = self.host.command([executable, '-S', 'DOCKER-USER'], False) + if result.returncode == 0 and any(line not in ('-N DOCKER-USER', '-A DOCKER-USER -j RETURN') + for line in result.stdout.splitlines()): + raise SecurityError('Unmanaged Docker firewall rules require reconciliation') + return {'version': 1, 'pristine': pristine, 'active': active, 'services': services, + 'files': files, 'helper_present': present, + 'ipv6': bool(re.search(r'^IPV6\s*=\s*[\"\x27]?yes', self._text('/etc/default/ufw'), re.M))} + + def _path(self, name): + path = self.host.path(name) + for parent in (path, *path.parents): + if parent.is_symlink(): + raise SecurityError('Refusing redirected security path: ' + name) + if not path.is_file(): + raise SecurityError('Required security file is missing: ' + name) + return path + + def _text(self, name): + return self._path(name).read_text() + + @staticmethod + def _digest(data): + return hashlib.sha256(data).hexdigest() + + def _files(self): + return {name: self._file(name) for name in FILES} + + def _file(self, name): + path = self._path(name) + data, metadata = path.read_bytes(), path.stat() + result = {'data': base64.b64encode(data).decode(), 'digest': self._digest(data), + 'mode': metadata.st_mode & 0o777, 'uid': metadata.st_uid, 'gid': metadata.st_gid} + result['fingerprint'] = self._digest(json.dumps(result, sort_keys=True).encode()) + return result + + def _service(self, name): + output = self.host.command(['systemctl', 'show', name, '-p', + 'LoadState,ActiveState,UnitFileState,FragmentPath,DropInPaths']).stdout + return dict(line.split('=', 1) for line in output.splitlines() if '=' in line) + + def _active(self): + output = self.host.command(['ufw', 'status']).stdout + if 'Status: inactive' in output: + return False + if 'Status: active' in output: + return True + raise SecurityError('Cannot determine UFW runtime state') + + def snapshot(self, plan, original_port, client_address): + if not isinstance(original_port, int) or not 1 <= original_port <= 65535: + raise SecurityError('Invalid bootstrap port') + try: + address = str(ipaddress.ip_address(client_address)) + except ValueError: + raise SecurityError('Invalid bootstrap peer address') from None + if plan != self.preflight(): + raise SecurityError('Security configuration changed after preflight') + if ipaddress.ip_address(address).version == 6 and not plan['ipv6']: + raise SecurityError('IPv6 bootstrap access requires enabled UFW IPv6 support') + if plan['services']['crowdsec']['ActiveState'] == 'active': + self._check_decisions(address) + return {'version': 1, 'plan': plan, 'original_port': original_port, + 'client_address': address, 'token': uuid.uuid4().hex, 'rules': [], + 'expected': {name: value['fingerprint'] for name, value in plan['files'].items()}, + 'service_changes': [], 'inflight': None, 'phase': 'prepared', + 'access_recovered': False, 'cleanup': 'not-started', 'helper_blocks': {}} + + def activate(self, state, persist): + if state['phase'] != 'prepared' or state['inflight']: + raise SecurityError('Security activation requires a fresh prepared snapshot') + self._assert_files(state) + state['phase'] = 'activating' + persist(state) + self._add_rule(state, state['original_port'], state['client_address'], True, persist) + self._add_rule(state, 2222, '0.0.0.0/0', False, persist) + if state['plan']['ipv6']: + self._add_rule(state, 2222, '::/0', False, persist) + if state['plan']['pristine']: + for policy, direction in (('deny', 'incoming'), ('allow', 'outgoing'), ('allow', 'routed')): + self._mutate(state, ['ufw', 'default', policy, direction], ['/etc/default/ufw'], persist) + if not state['plan']['helper_present']: + names = ['/etc/ufw/after.rules', '/etc/ufw/after6.rules'] + self._mutate(state, [HELPER, 'install'], names, persist) + for name in names: + blocks = BLOCK.findall(self._text(name)) + if len(blocks) != 1: + raise SecurityError('ufw-docker did not install a verifiable integration block') + state['helper_blocks'][name] = blocks[0] + persist(state) + self._check_helper_files() + self._mutate(state, ['ufw', '--force', 'enable'], ['/etc/ufw/ufw.conf'], persist) + self._mutate(state, ['ufw', 'reload'], [], persist) + self._enable(state, 'ufw', persist) + self.host.command(['crowdsec', '-t', '-c', '/etc/crowdsec/config.yaml']) + self._enable(state, 'crowdsec', persist) + self.host.command(['cscli', 'lapi', 'status']) + self._check_decisions(state['client_address']) + self._enable(state, 'crowdsec-firewall-bouncer', persist) + state['phase'] = 'active' + persist(state) + self.verify(state) + + def finalize(self, state, persist): + self.verify(state) + for rule in state['rules']: + if rule['temporary'] and rule['owned']: + self._delete_rule(state, rule, persist) + state['phase'] = 'final' + persist(state) + self.verify(state) + + def verify(self, state): + if state['phase'] not in ('active', 'final') or state['inflight']: + raise SecurityError('Security activation is incomplete') + self._assert_files(state) + if not self._active(): + raise SecurityError('UFW is not active') + for name in SERVICES: + service = self._service(name) + if service.get('ActiveState') != 'active' or service.get('UnitFileState') != 'enabled': + raise SecurityError('Security service is not active and enabled: ' + name) + self.host.command(['cscli', 'lapi', 'status']) + self._check_decisions(state['client_address']) + self._check_helper_files() + self._check_helper_runtime(state['plan']['ipv6']) + for rule in state['rules']: + if not (rule['temporary'] and state['phase'] == 'final') and self._rule_comment(rule) is None: + raise SecurityError('Required SSH firewall allowance is missing') + + def _check_helper_files(self): + for ipv6, name in ((False, 'after.rules'), (True, 'after6.rules')): + text = self._text('/etc/ufw/' + name) + blocks = BLOCK.findall(text) + if (len(blocks) != 1 or text.count('UFW AND DOCKER') != 2 + or [line.strip() for line in blocks[0].splitlines() if line.strip()] != _helper_lines(ipv6)): + raise SecurityError('Existing ufw-docker integration is custom or incomplete') + + def _check_helper_runtime(self, ipv6): + for version in (False, True) if ipv6 else (False,): + executable = 'ip6tables' if version else 'iptables' + prefix = 'ufw6' if version else 'ufw' + for chain in ('DOCKER-USER', prefix + '-docker-logging-deny'): + result = self.host.command([executable, '-S', chain], False) + expected = ['-N ' + chain] + [line for line in _helper_lines(version) + if line.startswith('-A ' + chain + ' ')] + try: + matches = (not result.returncode and [_runtime_rule(line) for line in result.stdout.splitlines()] + == [_runtime_rule(line) for line in expected]) + except ValueError: + matches = False + if not matches: + raise SecurityError('ufw-docker runtime chain is missing or changed: ' + executable + ' ' + chain) + if self.host.command([executable, '-C', 'FORWARD', '-j', 'DOCKER-USER'], False).returncode: + raise SecurityError('Docker forwarding hook is missing: ' + executable) + + def restore(self, state, persist): + state['cleanup'] = 'pending' + state['access_recovered'] = False + persist(state) + errors = [] + for change in reversed(state['service_changes']): + name, field = change['service'], change['field'] + if change['status'] == 'restored': + continue + if name == 'ufw' and field == 'ActiveState' and state['plan']['active']: + continue + try: + current = self._service(name) + if any(current.get(key) != change['metadata'].get(key) for key in change['metadata']): + raise SecurityError('Service unit changed during activation') + if current[field] == change['after']: + actions = {'active': 'start', 'inactive': 'stop', 'enabled': 'enable', 'disabled': 'disable'} + change['status'] = 'restoring' + persist(state) + self.host.command(['systemctl', actions[change['before']], name]) + elif current[field] != change['before']: + raise SecurityError('Service state changed outside this transition') + if self._service(name)[field] != change['before']: + raise SecurityError('Service state restoration did not converge') + change['status'] = 'restored' + persist(state) + except Exception: + errors.append('service:' + name + ':' + field) + ambiguous = [] + for name in FILES: + try: + if self._file(name)['fingerprint'] not in (state['plan']['files'][name]['fingerprint'], state['expected'][name]): + ambiguous.append(name) + except Exception: + ambiguous.append(name) + pending = bool(ambiguous or state['inflight'] or errors) + original_active = state['plan']['active'] + # Recover runtime access even when an unrelated edit prevents file cleanup. + try: + if not original_active: + try: + self._path('/etc/ufw/ufw.conf') + except SecurityError: + # A service stop removes runtime rules without following the replaced enable file. + self.host.command(['systemctl', 'stop', 'ufw']) + else: + self.host.command(['ufw', 'disable']) + state['access_recovered'] = not self._active() + elif pending: + self._retain_bootstrap(state, persist) + if not self._active(): + self.host.command(['ufw', '--force', 'enable']) + state['access_recovered'] = self._active() + except Exception: + errors.append('bootstrap-firewall') + for name, previous in state['plan']['files'].items(): + try: + path = self._path(name) + content = path.read_bytes() + fingerprint = self._file(name)['fingerprint'] + if fingerprint == previous['fingerprint']: + continue + if fingerprint == state['expected'][name] and not (original_active and pending and name.endswith(('user.rules', 'user6.rules'))): + atomic(path, base64.b64decode(previous['data']), previous['mode']) + os.chown(path, previous['uid'], previous['gid']) + continue + block = state['helper_blocks'].get(name) + if block and content.decode().count(block) == 1: + metadata = path.stat() + atomic(path, content.replace(block.encode(), b'', 1), metadata.st_mode & 0o777) + os.chown(path, metadata.st_uid, metadata.st_gid) + else: + errors.append('file:' + name) + except Exception: + errors.append('file:' + name) + # Rule removal is ownership-checked independently of whole-file snapshots. + for rule in state['rules']: + if rule['owned'] and not (original_active and pending and rule['temporary']): + try: + if self._rule_comment(rule) == rule['comment']: + self.host.command(self._rule_command(rule, delete=True)) + elif self._rule_comment(rule) is not None: + errors.append('rule-ownership') + except Exception: + errors.append('rule-cleanup') + try: + if original_active: + if errors or pending: + self._retain_bootstrap(state, persist) + else: + self.host.command(['ufw', 'reload']) + state['access_recovered'] = self._active() + else: + # Restoring the former ufw.conf must not reactivate the runtime. + state['access_recovered'] = not self._active() + except Exception: + errors.append('bootstrap-firewall') + if not state['access_recovered']: + errors.append('bootstrap-firewall') + if errors or pending: + state['diagnostics'] = sorted(set(errors + ['file:' + name for name in ambiguous] + + (['interrupted-mutation'] if state['inflight'] else []))) + persist(state) + raise SecurityError('Security cleanup pending; bootstrap firewall recovery=' + str(state['access_recovered']) + + '; inspect the root-owned recovery record') + state.update(cleanup='complete', phase='restored', inflight=None, diagnostics=[]) + persist(state) + + def _assert_files(self, state): + if any(value['fingerprint'] != state['expected'][name] for name, value in self._files().items()): + raise SecurityError('Security files changed outside this transition; recovery remains required') + + def _mutate(self, state, args, changed, persist): + self._assert_files(state) + state['inflight'] = {'command': args[:2], 'files': changed} + persist(state) + self.host.command(args) + current = self._files() + for name in changed: + state['expected'][name] = current[name]['fingerprint'] + if any(value['fingerprint'] != state['expected'][name] for name, value in current.items()): + raise SecurityError('Concurrent security file change during activation') + state['inflight'] = None + persist(state) + + def _enable(self, state, name, persist): + for field, before, after, operation in ( + ('UnitFileState', 'disabled', 'enabled', 'enable'), + ('ActiveState', 'inactive', 'active', 'start')): + current = self._service(name) + if current[field] == after: + continue + if current[field] != before: + raise SecurityError('Security service changed before activation: ' + name) + metadata = {key: current.get(key) for key in ('LoadState', 'FragmentPath', 'DropInPaths')} + if any(value != state['plan']['services'][name].get(key) for key, value in metadata.items()): + raise SecurityError('Security service unit changed before activation: ' + name) + change = {'service': name, 'field': field, 'before': before, 'after': after, + 'metadata': metadata, 'status': 'pending'} + state['service_changes'].append(change) + persist(state) + self.host.command(['systemctl', operation, name]) + if self._service(name)[field] != after: + raise SecurityError('Security service did not activate: ' + name) + change['status'] = 'applied' + persist(state) + + def _check_decisions(self, peer): + self.host.command(['cscli', 'lapi', 'status']) + try: + alerts = json.loads(self.host.command( + ['cscli', 'decisions', 'list', '--all', '--limit', '0', '--no-simu', '-o', 'json']).stdout) + if not isinstance(alerts, list): + raise ValueError() + address = ipaddress.ip_address(peer) + for alert in alerts: + if not isinstance(alert, dict) or not isinstance(alert.get('decisions'), list): + raise ValueError() + for decision in alert['decisions']: + if (not isinstance(decision, dict) or any(not isinstance(decision.get(key), str) + or not decision[key] for key in ('scope', 'value', 'type')) + or decision.get('simulated', False) is not False): + raise ValueError() + if decision['scope'].lower() not in ('ip', 'range'): + continue + network = ipaddress.ip_network(decision['value'], strict=False) + if address in network: + raise SecurityError('An existing CrowdSec decision covers the bootstrap peer') + except (ValueError, TypeError, KeyError): + raise SecurityError('Cannot verify CrowdSec decisions safely') from None + + @staticmethod + def _rule_file(rule): + return '/etc/ufw/user6.rules' if ipaddress.ip_network(rule['source'], strict=False).version == 6 else '/etc/ufw/user.rules' + + def _rule_comment(self, rule): + source = ipaddress.ip_network(rule['source'], strict=False) + any_address = '::/0' if source.version == 6 else '0.0.0.0/0' + for line in self._text(self._rule_file(rule)).splitlines(): + if not line.startswith('### tuple ### '): + continue + parts = line[len('### tuple ### '):].split() + if len(parts) < 7 or parts[:5] != ['allow', 'tcp', str(rule['port']), any_address, 'any'] or parts[6] != 'in': + continue + try: + if ipaddress.ip_network(parts[5], strict=False) != source: + continue + comment = next((p[8:] for p in parts[7:] if p.startswith('comment=')), '') + return bytes.fromhex(comment).decode() if comment else '' + except (ValueError, UnicodeError): + raise SecurityError('Cannot prove ownership of an existing SSH firewall rule') from None + return None + + @staticmethod + def _rule_command(rule, delete=False): + command = ['ufw', '--force', 'delete'] if delete else (['ufw', 'insert', '1'] if rule['temporary'] else ['ufw']) + return command + ['allow', 'proto', 'tcp', 'from', rule['source'], 'to', 'any', 'port', str(rule['port']), + 'comment', rule['comment']] + + def _add_rule(self, state, port, source, temporary, persist): + rule = {'port': port, 'source': source, 'temporary': temporary, + 'comment': 'apex-init-' + state['token'] + ('-bootstrap' if temporary else '-final')} + rule['owned'] = self._rule_comment(rule) is None + state['rules'].append(rule) + persist(state) + if rule['owned']: + self._mutate(state, self._rule_command(rule), ['/etc/ufw/user.rules', '/etc/ufw/user6.rules'], persist) + + def _delete_rule(self, state, rule, persist): + if self._rule_comment(rule) != rule['comment']: + raise SecurityError('Owned SSH rule changed; preserve it for explicit reconciliation') + self._mutate(state, self._rule_command(rule, delete=True), ['/etc/ufw/user.rules', '/etc/ufw/user6.rules'], persist) + + def _retain_bootstrap(self, state, persist): + rule = next((rule for rule in state['rules'] if rule['temporary']), None) + if rule is None: + raise SecurityError('No recorded bootstrap allowance exists') + if self._rule_comment(rule) is None: + rule['owned'] = True + state['bootstrap_retained'] = True + persist(state) + self.host.command(self._rule_command(rule)) diff --git a/engine/provisioning/seed.sh b/engine/provisioning/seed.sh new file mode 100755 index 0000000..533c136 --- /dev/null +++ b/engine/provisioning/seed.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Only the parser runtime may be installed before semantic input validation. +set -euo pipefail +export LC_ALL=C DEBIAN_FRONTEND=noninteractive +[[ $(id -u) == 0 ]] || { echo 'APEX seed requires root.' >&2; exit 1; } +[[ -r /etc/os-release ]] || { echo 'APEX seed requires Debian 13 amd64.' >&2; exit 1; } +. /etc/os-release +[[ ${ID:-} == debian && ${VERSION_ID:-} == 13 && $(dpkg --print-architecture) == amd64 ]] || { + echo 'APEX seed requires Debian 13 amd64.' >&2; exit 1; +} + +policy=/usr/sbin/policy-rc.d +state=/usr/sbin/.apex-policy-rc.d +for path in /var /var/lib /var/lib/apex-init; do + [[ ! -L $path ]] || { echo 'Refusing redirected initialization state.' >&2; exit 1; } +done +mkdir -p -m 0700 /var/lib/apex-init +[[ $(stat -c %u /var/lib/apex-init) == 0 ]] || { + echo 'Initialization state must be root-owned.' >&2; exit 1; +} +(( (8#$(stat -c %a /var/lib/apex-init) & 022) == 0 )) || { + echo 'Initialization state must not be writable by other users.' >&2; exit 1; +} +[[ ! -L /var/lib/apex-init/init.lock ]] || { echo 'Refusing redirected initialization lock.' >&2; exit 1; } +exec 8>/var/lib/apex-init/init.lock +flock -n 8 || { echo 'Another APEX initialization is running.' >&2; exit 1; } +mkdir -p /run/lock +exec 9>/run/lock/apex-package-policy.lock +flock -n 9 || { echo 'Another APEX package operation is running.' >&2; exit 1; } +restore_policy() { + [[ -d $state ]] || return 0 + [[ ! -L $state && $(stat -c %u "$state") == 0 ]] || { + echo 'Refusing untrusted service policy recovery state.' >&2; return 1; + } + if [[ -e $state/ready && ( -e $policy || -L $policy ) ]]; then + if ! cmp -s "$policy" <(printf '#!/bin/sh\nexit 101\n') && + ! cmp -s "$policy" "$state/original"; then + echo 'Service policy changed outside APEX; reconcile it explicitly.' >&2 + return 1 + fi + fi + if [[ -e $state/original || -L $state/original ]]; then + rm -f -- "$state/restoring" + cp -a -- "$state/original" "$state/restoring" + mv -Tf -- "$state/restoring" "$policy" + elif [[ -e $state/absent ]]; then + rm -f -- "$policy" + elif [[ -e $state/ready ]]; then + echo 'APEX service policy recovery is incomplete; inspect /usr/sbin/.apex-policy-rc.d.' >&2 + return 1 + fi + rm -rf -- "$state" +} + +# Release the shared host lock on exit; the target runner reacquires it later. +restore_policy +missing=() +for package in python3 python3-yaml; do + status=$(dpkg-query -W -f='${Status}' "$package" 2>/dev/null || true) + [[ $status == *' ok installed' ]] || missing+=("$package") +done +if (( ${#missing[@]} )); then + mkdir -m 0700 -- "$state" + trap restore_policy EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + if [[ -e $policy || -L $policy ]]; then + cp -a -- "$policy" "$state/original" + else + touch "$state/absent" + fi + touch "$state/ready" + printf '#!/bin/sh\nexit 101\n' > "$state/guard" + chmod 0755 "$state/guard" + mv -Tf -- "$state/guard" "$policy" + apt=(apt-get -o DPkg::Lock::Timeout=60 -o Acquire::Retries=2 + -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) + timeout 300 "${apt[@]}" update --error-on=any + install=("${apt[@]}" install -y --no-install-recommends --no-upgrade --no-remove "${missing[@]}") + plan=$(timeout 300 "${install[@]}" --simulate) + if grep -Eq '^Remv |^Inst [^ ]+ \[' <<< "$plan"; then + echo 'Parser installation requires a package upgrade/removal; reconcile it explicitly.' >&2 + exit 1 + fi + "${install[@]}" +fi +python3 -c 'import yaml' diff --git a/engine/provisioning/state.py b/engine/provisioning/state.py new file mode 100644 index 0000000..154e783 --- /dev/null +++ b/engine/provisioning/state.py @@ -0,0 +1,128 @@ +"""Durable enrollment records and writes that refuse redirected paths.""" +from __future__ import annotations + +import contextlib +import fcntl +import json +import os +from pathlib import Path +import stat +import tempfile +import time + +from . import ENROLLMENT_INTERFACE + + +class ProvisionError(RuntimeError): + pass + + +def safe_path(path): + path = Path(path).absolute() + for part in (*reversed(path.parents), path): + try: + info = part.lstat() + except FileNotFoundError: + continue + if stat.S_ISLNK(info.st_mode): + raise ProvisionError('managed path contains a symbolic link') + return path + + +def atomic_write(path, content, mode=0o600, uid=None, gid=None): + path = safe_path(path) + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and (not path.is_file() or path.stat().st_nlink != 1): + raise ProvisionError('managed file is not an ordinary unlinked file') + data = content.encode() if isinstance(content, str) else content + fd, temporary = tempfile.mkstemp(prefix='.apex-', dir=path.parent) + try: + with os.fdopen(fd, 'wb') as stream: + os.fchmod(stream.fileno(), mode) + if uid is not None: + os.fchown(stream.fileno(), uid, gid) + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +@contextlib.contextmanager +def exclusive_lock(path): + path = safe_path(path) + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ProvisionError('another initialization is running; inspect status before retrying') from None + yield + finally: + os.close(fd) + + +def check_heartbeat(path, now=None): + if path is None: + return + try: + age = (time.time() if now is None else now) - safe_path(path).stat().st_mtime + except FileNotFoundError: + raise ProvisionError('coordinator heartbeat is missing; stopping before the next stage') from None + if not 0 <= age < 90: + raise ProvisionError('coordinator heartbeat expired; stopping before the next stage') + + +class Journal: + def __init__(self, path, identity, revision, bundle_digest): + self.path = safe_path(path) + if self.path.exists(): + try: + self.data = json.loads(self.path.read_text()) + except (ValueError, OSError): + raise ProvisionError('invalid enrollment journal; reconcile it before retrying') from None + if self.data.get('enrollment_interface') != ENROLLMENT_INTERFACE: + raise ProvisionError('recorded enrollment interface is missing or unsupported') + if self.data.get('identity') != identity: + raise ProvisionError('enrollment identity conflicts with the recorded target') + if self.data.get('bundle_digest') != bundle_digest: + raise ProvisionError('provisioner bundle changed; resume using the recorded bundle') + else: + self.data = dict(identity=identity, revision=revision, bundle_digest=bundle_digest, + baseline_revision=revision, enrollment_interface=ENROLLMENT_INTERFACE, + status='running', stages={}) + + def save(self): + atomic_write(self.path, json.dumps(self.data, sort_keys=True, indent=2) + '\n') + + def stage(self, name, operation, heartbeat=None): + try: + check_heartbeat(heartbeat) + except ProvisionError: + self.data['status'] = 'stopped' + self.data['failed_stage'] = name + self.save() + raise + self.data['status'] = 'running' + self.data['stages'][name] = {'status': 'running'} + self.save() + try: + result = operation() + except Exception: + self.data['status'] = 'failed' + self.data['failed_stage'] = name + self.data['stages'][name] = {'status': 'failed'} + self.save() + raise + self.data['stages'][name] = {'status': 'done', 'result': result} + self.data.pop('failed_stage', None) + self.save() + return result diff --git a/engine/provisioning/storage.py b/engine/provisioning/storage.py new file mode 100644 index 0000000..7b9ce64 --- /dev/null +++ b/engine/provisioning/storage.py @@ -0,0 +1,508 @@ +"""Conservative target-side tier and optional swap reconciliation. + +The caller holds the initialization lock and supplies a zero-argument atomic +journal writer. Ambiguous device or pool ownership always requires remediation. +""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import tempfile + +from .contract import Enrollment + + +SRV = Path('/srv') +SWAPFILE = Path('/swapfile') +FSTAB = Path('/etc/fstab') +MEMINFO = Path('/proc/meminfo') +SWAPS = Path('/proc/swaps') +_OWNER = 'org.apex:enrollment' +_GIB = 1024 ** 3 + + +class StorageError(RuntimeError): + """Storage could not be positively identified or safely reconciled.""" + + +def _require(condition, message): + if not condition: + raise StorageError(message) + + +def _run(argv, *, full_vdev_paths=False): + environment = {**os.environ, 'LC_ALL': 'C'} + if full_vdev_paths: + environment.update(ZPOOL_VDEV_NAME_PATH='1', ZPOOL_VDEV_NAME_FOLLOW_LINKS='1') + environment.pop('ZPOOL_VDEV_NAME_GUID', None) + try: + result = subprocess.run(argv, check=False, capture_output=True, text=True, + timeout=300, env=environment) + except (OSError, subprocess.TimeoutExpired): + raise StorageError('required storage tool is unavailable or timed out: ' + argv[0]) from None + _require(result.returncode == 0, + 'storage inspection or operation failed: ' + ' '.join(argv[:2])) + return result.stdout + + +def _json(argv, key): + try: + value = json.loads(_run(argv))[key] + _require(isinstance(value, list), 'storage inspection returned an invalid list') + return value + except (ValueError, KeyError, TypeError): + raise StorageError('storage inspection returned unsupported JSON') from None + + +def _flatten(items): + for item in items: + yield item + yield from _flatten(item.get('children', [])) + + +def _mounts(): + return list(_flatten(_json(['findmnt', '--json', '--output', 'TARGET,SOURCE,FSTYPE,MAJ:MIN'], + 'filesystems'))) + + +def _device_number(path): + try: + info = os.stat(path) + except OSError: + raise StorageError('selected device is missing or is not a block device') from None + _require(stat.S_ISBLK(info.st_mode), 'selected device is not a block device') + return f'{os.major(info.st_rdev)}:{os.minor(info.st_rdev)}' + + +def _inventory(): + roots = _json(['lsblk', '--json', '--bytes', '--paths', '--output', + 'NAME,MAJ:MIN,TYPE,PKNAME,MOUNTPOINTS,FSTYPE,WWN,SERIAL,SIZE,PARTUUID'], 'blockdevices') + nodes, edges = {}, {} + + def visit(items, parent=None): + for item in items: + number = item['maj:min'] + nodes[number] = item + edges.setdefault(number, set()) + if parent: + edges[number].add(parent) + edges[parent].add(number) + visit(item.get('children', []), number) + + try: + visit(roots) + by_name = {item['name']: number for number, item in nodes.items()} + for number, item in nodes.items(): + parent = by_name.get(item.get('pkname')) + if parent: + edges[number].add(parent) + edges[parent].add(number) + except (KeyError, TypeError): + raise StorageError('block-device topology could not be established') from None + return nodes, edges + + +def _connected(number, edges): + found, pending = set(), [number] + while pending: + item = pending.pop() + if item not in found: + found.add(item) + pending.extend(edges.get(item, ())) + return found + + +def _pools(): + result = {} + for line in _run(['zpool', 'list', '-H', '-o', 'name,guid,health']).splitlines(): + fields = line.split('\t') + _require(len(fields) == 3 and fields[1].isdigit(), 'active pool identity is ambiguous') + result[fields[0]] = {'guid': fields[1], 'health': fields[2]} + return result + + +def _members(text, name, imported=True): + """Accept only a plain single-device ONLINE pool, never infer topology.""" + marker = re.search(r'^\s*config:\s*$', text, re.M) + _require(marker is not None, 'pool membership is unavailable') + lines = text[marker.end():].splitlines() + rows = [] + for line in lines: + fields = line.split() + if not fields or fields[0] == 'NAME': + continue + if fields[0] == 'errors:': + _require(line.strip() == 'errors: No known data errors', 'pool reports data errors') + break + _require(len(fields) >= 2 and fields[1] == 'ONLINE', + 'pool is not a healthy single-device topology') + if imported: + _require(len(fields) == 5 and fields[2:] == ['0', '0', '0'], + 'pool reports device errors') + rows.append(fields[0]) + _require(len(rows) == 2 and rows[0] == name, + 'pool must contain exactly one identifiable device') + leaf = rows[1] + if not imported and re.fullmatch(r'[A-Za-z0-9_.-]+', leaf): + leaf = '/dev/' + leaf + _require(leaf.startswith('/dev/'), 'pool device path cannot be positively resolved') + return leaf + + +def _pool_device(name): + return _members(_run(['zpool', 'status', '-P', name], full_vdev_paths=True), name) + + +def _inspect_device(device, mounts, allowed_pool=None, allowed_path=None): + number = _device_number(device) + nodes, edges = _inventory() + _require(number in nodes, 'selected device is absent from the block topology') + node = nodes[number] + hardware = node.get('wwn') or node.get('serial') + partition = None + if node.get('type') == 'part': + parents = [item for item in nodes.values() if item['name'] == node.get('pkname')] + _require(len(parents) == 1 and parents[0]['type'] == 'disk' and node.get('partuuid'), + 'selected partition needs a stable partition UUID and identifiable parent disk') + hardware = parents[0].get('wwn') or parents[0].get('serial') + partition = node['partuuid'] + _require(isinstance(hardware, str) and bool(hardware.strip()), + 'selected device needs a stable WWN or hardware serial') + _require(node.get('type') in ('disk', 'part') and int(node.get('size') or 0) > 0, + 'selected device type or capacity is unsupported') + related = _connected(number, edges) + protected = set() + for other, data in nodes.items(): + for mount in data.get('mountpoints') or []: + if mount and mount != allowed_path: + protected.add(other) + for mount in mounts: + if mount['target'] != allowed_path: + protected.add(mount.get('maj:min')) + active = _pools() + for pool in active: + leaf = _device_number(_pool_device(pool)) + if pool != allowed_pool or any( + m['source'].split('/')[0] == pool and m['target'] != allowed_path for m in mounts): + protected.add(leaf) + _require(not (related & protected), + 'selected device intersects mounted, root, boot, swap, or active pool storage') + identity = {'hardware': hardware, 'size': int(node['size']), 'type': node['type']} + if partition: + identity['partition'] = partition + return {'number': number, 'identity': identity, 'related': related, 'node': node, 'pools': active} + + +def _safe_path(path): + for parent in reversed((path, *path.parents)): + try: + info = parent.lstat() + except FileNotFoundError: + _require(parent == path, 'storage parent directory is missing') + continue + _require(stat.S_ISDIR(info.st_mode), 'storage path contains a symlink or non-directory') + + +def _directory(path, mounts, pool=None): + _safe_path(path) + exact = [m for m in mounts if m['target'] == str(path)] + _require(not exact or (len(exact) == 1 and pool is not None and + exact[0]['fstype'] == 'zfs' and exact[0]['source'] == pool), + 'tier path is occupied by an unexpected mount') + _require(not any(m['target'].startswith(str(path) + '/') for m in mounts), + 'tier path contains an unexpected nested mount') + created = not path.exists() + if created: + path.mkdir(mode=0o755) + return created + + +def _directory_identity(path): + _safe_path(path) + try: + info = path.lstat() + except FileNotFoundError: + raise StorageError('managed directory identity is missing') from None + return {'device': info.st_dev, 'inode': info.st_ino} + + +def _verify_ownership_identity(record, path): + ownership = record.get('ownership') + if ownership: + _require(ownership['identity'] == _directory_identity(path), + 'managed directory identity changed before ownership handoff') + return ownership + + +def _record_pending_ownership(record, path): + if not _verify_ownership_identity(record, path): + record['ownership'] = {'status': 'pending', 'identity': _directory_identity(path)} + + +def _verify_pool(name, guid, device_info, path, owner, require_owner=True): + pools = _pools() + _require(name in pools and pools[name] == {'guid': guid, 'health': 'ONLINE'}, + 'pool name, GUID, or health does not match enrollment') + _require(_member_matches(_pool_device(name), device_info), + 'pool device membership does not match the selected device') + rows = [line.split('\t') for line in _run( + ['zfs', 'list', '-H', '-r', '-o', 'name,mountpoint,type,mounted', name]).splitlines()] + _require(len(rows) == 1 and len(rows[0]) == 4 and rows[0][:3] == [name, str(path), 'filesystem'], + 'pool dataset or mountpoint requires explicit reconciliation') + if require_owner: + actual = _run(['zfs', 'get', '-H', '-o', 'value', _OWNER, name]).strip() + _require(actual == owner, 'pool lacks matching enrollment ownership') + return rows[0][3] == 'yes' + + +def _member_matches(member, info): + number = _device_number(member) + return number == info['number'] or (info['node']['type'] == 'disk' and number in info['related']) + + +def _import_candidate(device, guid): + text = _run(['zpool', 'import', '-d', device], full_vdev_paths=True) + names = re.findall(r'^\s*pool:\s*(\S+)\s*$', text, re.M) + guids = re.findall(r'^\s*id:\s*(\d+)\s*$', text, re.M) + states = re.findall(r'^\s*state:\s*(\S+)\s*$', text, re.M) + _require(len(names) == 1 and guids == [guid] and states == ['ONLINE'], + 'import discovery does not match the declared healthy pool GUID') + actions = re.findall(r'^\s*action:\s*(.+)$', text, re.M) + _require(not re.search(r'^\s*status:', text, re.M) and + all(action == 'The pool can be imported using its name or numeric identifier.' for action in actions), + 'pool import reports ownership or recovery conditions; reconcile manually') + return names[0], _members(text, names[0], imported=False) + + +def _block_tier(tier, path, journal, owner, save): + record = journal.get(str(tier.number)) + created = False + pool = record.get('pool') if record else None + info = _inspect_device(tier.device, _mounts(), pool, str(path) if pool else None) + if record: + _require(record['device'] == info['identity'], 'selected device identity changed since enrollment') + _require(record['status'] in ('intent', 'complete'), 'tier journal status is unsupported') + if pool and pool in info['pools']: + guid = record.get('guid') or info['pools'][pool]['guid'] + _verify_pool(pool, guid, info, path, owner) + else: + _require(not record or record['status'] == 'intent', + 'previously enrolled pool is not active; explicit reconciliation is required') + if tier.mode == 'create': + pool = f'tier-{tier.number}' + _require(pool not in info['pools'], 'existing pool name does not prove enrollment ownership') + _require(info['node']['type'] == 'disk', 'pool creation requires an unused whole disk') + _require(len(info['related']) == 1, 'selected disk has partitions or dependent devices') + signatures = _json(['wipefs', '--json', '--output', 'TYPE,UUID,LABEL', tier.device], 'signatures') + _require(not signatures and not info['node'].get('fstype'), 'selected device contains existing signatures') + else: + pool, member = _import_candidate(tier.device, tier.pool_guid) + _require(_member_matches(member, info), 'import membership differs from the selected device') + _require(pool not in info['pools'], 'pool is already active without matching enrollment ownership') + created = _directory(path, _mounts()) + _require(not path.exists() or not any(path.iterdir()), 'pool mountpoint contains existing data') + record = {'status': 'intent', 'device': info['identity'], 'pool': pool, + 'ownership_intent': tier.mode == 'create'} + journal[str(tier.number)] = record + save() + fresh = _inspect_device(tier.device, _mounts()) + _require(fresh['identity'] == info['identity'] and fresh['number'] == info['number'], + 'selected device identity changed before pool operation') + _directory(path, _mounts()) + _require(not any(path.iterdir()), 'pool mountpoint acquired existing data before operation') + if tier.mode == 'create': + _require(not _json(['wipefs', '--json', '--output', 'TYPE,UUID,LABEL', tier.device], 'signatures'), + 'selected device acquired signatures before creation') + _run(['zpool', 'create', '-o', 'ashift=12', '-O', f'mountpoint={path}', + '-O', f'{_OWNER}={owner}', pool, tier.device]) + else: + _require(_import_candidate(tier.device, tier.pool_guid) == (pool, member), + 'pool identity changed before import') + _run(['zpool', 'import', '-N', '-d', tier.device, tier.pool_guid]) + pools = _pools() + _require(pool in pools, 'pool operation did not establish an active pool') + guid = pools[pool]['guid'] + _require(tier.mode != 'import' or guid == tier.pool_guid, 'imported pool GUID differs from intent') + refreshed = _inspect_device(tier.device, _mounts(), pool, str(path)) + _require(refreshed['identity'] == info['identity'], 'selected device changed during pool operation') + info = refreshed + _verify_pool(pool, guid, info, path, owner, require_owner=tier.mode == 'create') + if tier.mode == 'import': + previous_owner = _run(['zfs', 'get', '-H', '-o', 'value', _OWNER, pool]).strip() + _require(previous_owner in ('-', owner), 'imported pool belongs to a different enrollment') + _run(['zfs', 'set', f'{_OWNER}={owner}', pool]) + created = _directory(path, _mounts(), pool) or created + if not _verify_pool(pool, guid, info, path, owner): + _require(not any(path.iterdir()), 'refusing to hide existing files beneath a pool mount') + _run(['zfs', 'mount', pool]) + _require(any(m['target'] == str(path) and m['source'] == pool and m['fstype'] == 'zfs' + for m in _mounts()), 'pool is not mounted at its enrolled path') + record.update(status='complete', guid=guid) + if tier.mode == 'create' and record.get('ownership_intent'): + _record_pending_ownership(record, path) + save() + return {'path': str(path), 'mode': tier.mode, 'created': created and tier.mode == 'create', + 'pool': pool, 'guid': guid} + + +def _persist_swap(): + info = FSTAB.lstat() + _require(stat.S_ISREG(info.st_mode) and info.st_uid == 0, 'fstab must be a root-owned regular file') + original = FSTAB.read_text() + lines = [line.split() for line in original.splitlines() if line.strip() and not line.lstrip().startswith('#')] + matches = [line for line in lines if line[0] == str(SWAPFILE)] + if matches: + _require(len(matches) == 1 and len(matches[0]) >= 4 and matches[0][2] == 'swap' + and 'noauto' not in matches[0][3].split(','), + 'existing swap fstab entry is incompatible') + return + fd, temporary = tempfile.mkstemp(prefix='.apex-swap-', dir=FSTAB.parent) + try: + with os.fdopen(fd, 'w') as output: + os.fchmod(output.fileno(), stat.S_IMODE(info.st_mode)) + os.fchown(output.fileno(), info.st_uid, info.st_gid) + output.write(original + ('' if original.endswith('\n') else '\n') + + f'{SWAPFILE} none swap sw 0 0\n') + output.flush() + os.fsync(output.fileno()) + current = FSTAB.lstat() + _require(all(getattr(current, key) == getattr(info, key) for key in + ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')), + 'fstab changed during swap setup') + os.replace(temporary, FSTAB) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def _active_swaps(): + lines = SWAPS.read_bytes().splitlines() + _require(bool(lines) and lines[0].split() == [b'Filename', b'Type', b'Size', b'Used', b'Priority'], + 'kernel swap inventory header is invalid') + paths = [] + for line in lines[1:]: + fields = line.split() + _require(len(fields) == 5 and fields[0].startswith(b'/') and fields[1] in (b'file', b'partition') + and fields[2].isdigit() and fields[3].isdigit() + and re.fullmatch(rb'-?[0-9]+', fields[4]) is not None, + 'kernel swap inventory entry is invalid') + _require(re.search(rb'\\(?![0-3][0-7]{2})', fields[0]) is None, + 'kernel swap inventory path escaping is invalid') + decoded = re.sub(rb'\\([0-3][0-7]{2})', lambda match: bytes([int(match.group(1), 8)]), fields[0]) + paths.append(os.fsdecode(decoded)) + return paths + + +def _swap(journal, save): + try: + active = _active_swaps() + if active: + return {'status': 'skipped', 'reason': 'existing active swap preserved'} + if os.path.lexists(SWAPFILE): + return {'status': 'skipped', 'reason': 'existing swapfile preserved; no overwrite or resize'} + mounts = _mounts() + covering = [m for m in mounts if str(SWAPFILE).startswith(m['target'].rstrip('/') + '/')] + _require(bool(covering), 'swap filesystem could not be identified') + filesystem = max(covering, key=lambda m: len(m['target']))['fstype'] + _require(filesystem in ('ext4', 'xfs'), 'optional swap requires a supported ext4 or XFS filesystem') + match = re.search(r'^MemTotal:\s+(\d+)\s+kB$', MEMINFO.read_text(), re.M) + _require(match is not None, 'RAM size could not be established') + size = (4 if int(match.group(1)) * 1024 > 4 * _GIB else 2) * _GIB + space = os.statvfs(SWAPFILE.parent) + _require(space.f_bavail * space.f_frsize >= size + _GIB, + 'insufficient free space for optional swap and reserve') + _require(hasattr(os, 'posix_fallocate'), 'safe swap allocation is unavailable') + journal['swap'] = {'status': 'intent', 'size': size} + save() + fd = os.open(SWAPFILE, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + os.posix_fallocate(fd, 0, size) + os.fsync(fd) + finally: + os.close(fd) + _run(['mkswap', str(SWAPFILE)]) + _run(['swapon', str(SWAPFILE)]) + active = _active_swaps() + _require(str(SWAPFILE) in active, + 'optional swap activation did not become observable') + _persist_swap() + return {'status': 'complete', 'size': size} + except (StorageError, OSError) as error: + return {'status': 'skipped', 'reason': str(error) if isinstance(error, StorageError) + else 'optional swap allocation or persistence failed; existing resources preserved'} + + +def _enrollment_identity(spec): + return hashlib.sha256(json.dumps(spec.identity(), sort_keys=True, + separators=(',', ':')).encode()).hexdigest() + + +def acknowledge_storage_ownership(spec: Enrollment, state: dict, paths, save) -> None: + """Persist the caller's successful, nonrecursive ownership handoff for paths.""" + journal = state.get('storage', {}) + _require(journal.get('identity') == _enrollment_identity(spec), + 'storage enrollment identity conflicts with ownership acknowledgement') + allowed = {str(SRV / f'tier-{tier.number}.{spec.repository.name}'): tier for tier in spec.tiers} + acknowledged = [] + for path in paths: + _require(path in allowed, 'ownership acknowledgement path is outside enrollment') + tier = allowed[path] + record = journal.get('tiers', {}).get(str(tier.number), {}) + ownership = _verify_ownership_identity(record, Path(path)) + _require(tier.mode != 'import' and ownership is not None, + 'ownership acknowledgement requires a directory created by enrollment') + _require(ownership['status'] in ('pending', 'applied'), 'ownership journal status is unsupported') + acknowledged.append(ownership) + for ownership in acknowledged: + ownership['status'] = 'applied' + save() + + +def reconcile_storage(spec: Enrollment, state: dict, save) -> dict: + """Return ownership_pending paths until the caller acknowledges their chown.""" + identity = _enrollment_identity(spec) + journal = state.setdefault('storage', {'identity': identity, 'tiers': {}}) + _require(journal.get('identity') == identity, 'storage enrollment identity conflicts with journal') + _safe_path(SRV) + _require(SRV.exists(), 'storage parent directory is missing') + selected = set() + for tier in spec.tiers: + if tier.device: + number = _device_number(tier.device) + _require(number not in selected, 'multiple tiers select aliases of the same device') + selected.add(number) + result = {'tiers': [], 'ownership_pending': []} + for tier in spec.tiers: + path = SRV / f'tier-{tier.number}.{spec.repository.name}' + if tier.mode == 'directory': + record = journal['tiers'].get(str(tier.number), {}) + _verify_ownership_identity(record, path) + _safe_path(path) + _require(not (record.get('ownership_intent') and not record.get('ownership') and path.exists()), + 'directory creation was interrupted before identity was recorded; reconcile manually') + if not path.exists(): + record = {'status': 'intent', 'path': str(path), 'ownership_intent': True} + journal['tiers'][str(tier.number)] = record + save() + created = _directory(path, _mounts()) + if created: + _record_pending_ownership(record, path) + entry = {'path': str(path), 'mode': tier.mode, 'created': created} + record.update(status='complete', path=str(path)) + journal['tiers'][str(tier.number)] = record + save() + else: + entry = _block_tier(tier, path, journal['tiers'], identity, save) + ownership = journal['tiers'][str(tier.number)].get('ownership') + if ownership and ownership['status'] == 'pending': + result['ownership_pending'].append(str(path)) + result['tiers'].append(entry) + result['swap'] = _swap(journal, save) + journal['swap'] = result['swap'] + save() + return result diff --git a/engine/provisioning/target.py b/engine/provisioning/target.py new file mode 100644 index 0000000..2a654ba --- /dev/null +++ b/engine/provisioning/target.py @@ -0,0 +1,170 @@ +"""Shared root runner for remote and cloud-init enrollment.""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import platform +import pwd +import subprocess +import sys +import tempfile + +from .state import Journal, ProvisionError, atomic_write, exclusive_lock, safe_path +from .userdata import decode_user_data + +STATE = Path('/var/lib/apex-init') + + +def run(args, **kwargs): + result = subprocess.run(args, text=True, capture_output=True, timeout=300, **kwargs) + if result.returncode: + raise ProvisionError('required target operation failed: ' + args[0]) + return result.stdout.strip() + + +def preflight(): + if os.geteuid() != 0: + raise ProvisionError('target enrollment requires root privileges') + release = {} + for line in Path('/etc/os-release').read_text().splitlines(): + key, separator, value = line.partition('=') + if separator: + release[key] = value.strip('"') + if release.get('ID') != 'debian' or release.get('VERSION_ID') != '13' or platform.machine() != 'x86_64': + raise ProvisionError('initialization supports Debian 13 amd64 only') + + +def validate_keys(spec): + for key in spec.authorized_keys: + with tempfile.NamedTemporaryFile(mode='w') as stream: + stream.write(key + '\n') + stream.flush() + try: + run(['ssh-keygen', '-l', '-f', stream.name]) + except ProvisionError: + raise ProvisionError('administrator public key failed cryptographic validation') from None + + +def administrator(spec): + try: + account = pwd.getpwnam('adam') + except KeyError: + run(['useradd', '--create-home', '--shell', '/bin/bash', '--groups', 'sudo,docker', 'adam']) + account = pwd.getpwnam('adam') + if account.pw_dir != '/home/adam' or account.pw_uid == 0: + raise ProvisionError('existing administrator has incompatible home or UID') + safe_path(account.pw_dir) + if account.pw_shell != '/bin/bash': + run(['usermod', '--shell', '/bin/bash', 'adam']) + run(['usermod', '--append', '--groups', 'sudo,docker', 'adam']) + directory = safe_path(Path(account.pw_dir) / '.ssh') + directory.mkdir(mode=0o700, exist_ok=True) + os.chown(directory, account.pw_uid, account.pw_gid) + os.chmod(directory, 0o700) + authorized = safe_path(directory / 'authorized_keys') + existing = authorized.read_text().splitlines() if authorized.exists() else [] + keys = existing + [key for key in spec.authorized_keys if key not in existing] + atomic_write(authorized, '\n'.join(keys) + '\n', uid=account.pw_uid, gid=account.pw_gid) + policy = 'adam ALL=(ALL:ALL) NOPASSWD: ALL\n' + with tempfile.NamedTemporaryFile(mode='w') as stream: + stream.write(policy) + stream.flush() + run(['visudo', '-cf', stream.name]) + atomic_write('/etc/sudoers.d/99-apex-adam', policy, 0o440) + run(['visudo', '-c']) + run(['runuser', '-u', 'adam', '--', 'sudo', '-n', 'true']) + for item in spec.files: + uid, gid = (0, 0) if item.owner == 'root:root' else (account.pw_uid, account.pw_gid) + atomic_write(item.path, item.content, item.mode, uid=uid, gid=gid) + if run(['hostname']) != spec.hostname: + run(['hostnamectl', 'set-hostname', spec.hostname]) + groups = run(['id', '-nG', 'adam']).split() + if not {'sudo', 'docker'} <= set(groups): + raise ProvisionError('administrator groups did not converge') + return {'administrator': 'adam', 'hostname': spec.hostname, 'sudo': 'verified'} + + +def enroll(spec, revision, bundle_digest, heartbeat=None): + from .baseline import apply_baseline + from .storage import acknowledge_storage_ownership, reconcile_storage + from .repository import reconcile_repository + + preflight() + validate_keys(spec) + safe_path(STATE).mkdir(mode=0o700, exist_ok=True) + if STATE.stat().st_uid != 0: + raise ProvisionError('enrollment state directory must be root-owned') + os.chmod(STATE, 0o700) + with exclusive_lock(STATE / 'init.lock'): + journal = Journal(STATE / 'enrollment.json', spec.identity(), revision, bundle_digest) + journal.save() + journal.stage('baseline', apply_baseline, heartbeat) + journal.stage('administrator', lambda: administrator(spec), heartbeat) + result = journal.stage('storage', lambda: reconcile_storage(spec, journal.data, journal.save), heartbeat) + account = pwd.getpwnam('adam') + for path in result['ownership_pending']: + os.chown(safe_path(path), account.pw_uid, account.pw_gid) + acknowledge_storage_ownership(spec, journal.data, [path], journal.save) + journal.stage('repository', lambda: reconcile_repository(spec, journal.data, journal.save), heartbeat) + journal.data['status'] = 'enrolled' + journal.data['external_access'] = 'pending' + journal.save() + return journal.data + + +def main(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument('operation', choices=('validate', 'enroll', 'status', 'complete')) + parser.add_argument('--user-data') + parser.add_argument('--revision', default='unknown') + parser.add_argument('--bundle-digest', default='unknown') + parser.add_argument('--heartbeat') + args = parser.parse_args(argv) + try: + preflight() + if args.operation == 'status': + path = safe_path(STATE / 'enrollment.json') + print(path.read_text() if path.exists() else json.dumps({'status': 'not-started'})) + return 0 + if args.operation == 'complete': + with exclusive_lock(STATE / 'init.lock'): + access = json.loads(safe_path(STATE / 'access/state.json').read_text()) + path = safe_path(STATE / 'enrollment.json') + record = json.loads(path.read_text()) + if access.get('status') != 'committed' or record.get('status') not in ('enrolled', 'complete'): + raise ProvisionError('enrollment and access must be verified before completion') + record['external_access'] = 'verified' + record['status'] = 'complete' + atomic_write(path, json.dumps(record, sort_keys=True) + '\n') + print(json.dumps(record, sort_keys=True)) + return 0 + if not args.user_data: + raise ProvisionError('--user-data is required') + path = safe_path(args.user_data) + if path.stat().st_uid != 0 or path.stat().st_mode & 0o077: + raise ProvisionError('user-data must be root-owned and private') + if not path.is_file() or path.stat().st_size > 1024 * 1024: + raise ProvisionError('user-data must be a regular file no larger than 1 MiB') + spec = decode_user_data(path.read_bytes()) + validate_keys(spec) + if args.operation == 'validate': + print(json.dumps({'status': 'valid', 'hostname': spec.hostname})) + else: + print(json.dumps(enroll(spec, args.revision, args.bundle_digest, args.heartbeat), sort_keys=True)) + return 0 + except Exception as error: + from .contract import ContractError + from .baseline import BaselineError + from .storage import StorageError + if isinstance(error, (ProvisionError, ContractError, BaselineError, StorageError)): + print(str(error), file=sys.stderr) + else: + # Unexpected dependency failures must not render raw credential input. + print('initialization failed (' + type(error).__name__ + '); inspect stage status', file=sys.stderr) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/engine/provisioning/transport.py b/engine/provisioning/transport.py new file mode 100644 index 0000000..6ee1d38 --- /dev/null +++ b/engine/provisioning/transport.py @@ -0,0 +1,385 @@ +"""Workstation SSH transport. User-data stays opaque until target validation.""" +from __future__ import annotations + +import hashlib +import io +import ipaddress +import json +import os +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import tarfile +import tempfile +import time +import uuid + + +class TransportError(RuntimeError): + pass + + +class HandoverUncertain(TransportError): + pass + + +# This runs before Python is guaranteed to exist on the Debian target. All +# ancestors must be trusted; a root-owned sticky temporary directory is safe +# only because root owns each managed child and exclusive mktemp creates it. +REMOTE_PATH_GUARD = r''' +set -eu +umask 077 +apex_guard_dir() { + apex_checked=$1 + while :; do + test ! -L "$apex_checked" && test -d "$apex_checked" || exit 91 + test "$(stat -c %u -- "$apex_checked")" = 0 || exit 91 + apex_mode=$(stat -c %a -- "$apex_checked") + test "$((0$apex_mode & 0022))" = 0 || test "$((0$apex_mode & 01000))" != 0 || exit 91 + test "$apex_checked" != / || break + apex_checked=${apex_checked%/*} + test -n "$apex_checked" || apex_checked=/ + done +} +apex_ensure_dir() { + apex_new_dir=$1 + if test -e "$apex_new_dir" || test -L "$apex_new_dir"; then + apex_guard_dir "$apex_new_dir" + apex_mode=$(stat -c %a -- "$apex_new_dir") + test "$((0$apex_mode & 0022))" = 0 || exit 91 + else + apex_guard_dir "${apex_new_dir%/*}" + mkdir -m "$2" -- "$apex_new_dir" + apex_guard_dir "$apex_new_dir" + fi +} +apex_guard_file() { + apex_file=$1 + apex_guard_dir "${apex_file%/*}" + apex_mode=$(stat -c %a -- "${apex_file%/*}") + test "$((0$apex_mode & 0022))" = 0 || exit 91 + test ! -L "$apex_file" || exit 91 + if test -e "$apex_file"; then + test -f "$apex_file" || exit 91 + test "$(stat -c %u -- "$apex_file")" = 0 || exit 91 + test "$(stat -c %h -- "$apex_file")" = 1 || exit 91 + apex_mode=$(stat -c %a -- "$apex_file") + test "$((0$apex_mode & 0022))" = 0 || exit 91 + fi +} +apex_atomic_input() { + apex_destination=$1 + apex_guard_file "$apex_destination" + apex_temporary=$(mktemp "${apex_destination%/*}/.apex-write.XXXXXXXXXX") + trap 'rm -f -- "$apex_temporary"' EXIT HUP INT TERM + cat > "$apex_temporary" + chmod 0600 "$apex_temporary" + apex_guard_file "$apex_destination" + mv -T -- "$apex_temporary" "$apex_destination" + trap - EXIT HUP INT TERM +} +''' + + +def remote_atomic_input(path): + return REMOTE_PATH_GUARD + '\napex_atomic_input ' + shlex.quote(path) + + +def remote_bundle_install(stage, installed, digest): + return REMOTE_PATH_GUARD + '\n' + ( + 'apex_guard_dir ' + shlex.quote(stage) + '; apex_guard_file ' + shlex.quote(stage + '/bundle.tar') + '; ' + 'test "$(sha256sum ' + shlex.quote(stage + '/bundle.tar') + ' | cut -d " " -f 1)" = ' + digest + '; ' + 'apex_ensure_dir /usr/local/lib/apex-provisioner 0755; ' + 'if test -e ' + installed + ' || test -L ' + installed + '; then ' + 'apex_guard_dir ' + installed + '; ' + 'apex_bad=$(find ' + installed + ' -xdev \\( -type l -o ! -user root -o -perm /022 -o \\( ! -type d -a ! -type f \\) \\) -print -quit); test -z "$apex_bad"; ' + 'else apex_install_tmp=$(mktemp -d /usr/local/lib/apex-provisioner/.install.XXXXXXXXXX); ' + 'trap \'rm -rf -- "$apex_install_tmp"\' EXIT HUP INT TERM; ' + 'tar --no-same-owner --same-permissions -xf ' + stage + '/bundle.tar -C "$apex_install_tmp"; ' + 'mv -T -- "$apex_install_tmp" ' + installed + '; trap - EXIT HUP INT TERM; fi; ' + 'tar --compare -f ' + stage + '/bundle.tar -C ' + installed) + + +def local_file(value, name): + if not value or any(ord(c) < 32 for c in str(value)): + raise TransportError(f'{name} must name a readable regular file') + try: + path = Path(value).expanduser().resolve(strict=True) + except (OSError, ValueError): + raise TransportError(f'{name} must name a readable regular file') from None + if not path.is_file() or not os.access(path, os.R_OK): + raise TransportError(f'{name} must name a readable regular file') + return path + + +def destination(value, port=None): + match = re.fullmatch(r'([A-Za-z_][A-Za-z0-9_.-]{0,31})@([A-Za-z0-9][A-Za-z0-9_.:-]*|\[[0-9a-fA-F:]+\])', value or '') + if not match or '..' in match[2]: + raise TransportError('Target must be USER@HOST without SSH options; use root or an existing administrator with sudo -n') + hostname = match[2].strip('[]') + if ':' in hostname: + try: + ipaddress.IPv6Address(hostname) + except ValueError: + raise TransportError('IPv6 targets must contain a valid address; pass ports separately') from None + if port is not None and (not re.fullmatch(r'[0-9]{1,5}', str(port)) or not 1 <= int(port) <= 65535): + raise TransportError('Port must be an integer from 1 through 65535') + return match[1], hostname, int(port) if port is not None else None + + +def build_bundle(commons): + root = Path(commons) + files = sorted((root / 'engine').rglob('*.py')) + seed = root / 'engine/provisioning/seed.sh' + if not seed.is_file(): + raise TransportError('Provisioning seed is missing') + files.append(seed) + output = io.BytesIO() + with tarfile.open(fileobj=output, mode='w') as archive: + for path in files: + if path.is_symlink() or '__pycache__' in path.parts: + raise TransportError('Provisioning bundle contains an unsafe source path') + data = path.read_bytes() + info = tarfile.TarInfo(path.relative_to(root).as_posix()) + info.size, info.mode, info.mtime = len(data), (0o755 if path == seed else 0o644), 0 + info.uid = info.gid = 0 + archive.addfile(info, io.BytesIO(data)) + data = output.getvalue() + revision = 'unversioned-modified' + if shutil.which('git'): + result = subprocess.run(['git', '-C', str(root), 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=30) + if result.returncode == 0 and re.fullmatch(r'[a-f0-9]{40,64}\n?', result.stdout): + revision = result.stdout.strip() + dirty = subprocess.run(['git', '-C', str(root), 'status', '--porcelain', '--', 'engine'], capture_output=True, text=True, timeout=30) + if dirty.returncode or dirty.stdout.strip(): + revision += '-modified' + return data, hashlib.sha256(data).hexdigest(), revision + + +class SSH: + def __init__(self, target, port=None, identity_file=None, runner=subprocess.run): + self.user, self.host, self.port = destination(target, port) + self.identity = local_file(identity_file, 'Identity file') if identity_file else None + self.runner = runner + self.resolved_host = None + self.resolved_port = None + self.trust_file = None + + def arguments(self, fresh=False): + args = ['ssh', '-T', '-o', 'StrictHostKeyChecking=yes', '-o', 'ConnectTimeout=15', + '-o', 'ServerAliveInterval=10', '-o', 'ServerAliveCountMax=3'] + if self.identity: + args += ['-i', str(self.identity)] + args += ['-l', 'adam' if fresh else self.user] + if fresh: + if not self.trust_file or not self.resolved_host: + raise TransportError('Fresh access requires the verified bootstrap host identity') + args += ['-p', '2222', '-o', 'ControlMaster=no', '-o', 'ControlPath=none', + '-o', 'ControlPersist=no', '-o', 'BatchMode=yes', + '-o', 'PreferredAuthentications=publickey', '-o', 'PasswordAuthentication=no', + '-o', 'KbdInteractiveAuthentication=no', '-o', 'Hostname=' + self.resolved_host, + '-o', 'HostKeyAlias=apex-init-verified-target', + '-o', 'UserKnownHostsFile=' + str(self.trust_file), + '-o', 'GlobalKnownHostsFile=/dev/null'] + elif self.port: + args += ['-p', str(self.port)] + return args + + def resolve(self): + result = self.runner(self.arguments() + ['-G', '--', self.host], capture_output=True, text=True, timeout=30) + if result.returncode: + raise TransportError('Unable to resolve SSH connection configuration') + values = dict(line.split(' ', 1) for line in result.stdout.splitlines() if ' ' in line) + hostname = values.get('hostname', '') + port = values.get('port', '') + destination(self.user + '@' + hostname, port) + self.resolved_host, self.resolved_port = hostname, int(port) + + def run(self, script, data=None, fresh=False, privileged=True, check=True, timeout=300): + command = ['sh', '-c', script] + if privileged and (fresh or self.user != 'root'): + command = ['sudo', '-n', '--', *command] + result = self.runner(self.arguments(fresh) + ['--', self.host, shlex.join(command)], + input=data, capture_output=True, timeout=timeout) + if check and result.returncode: + # SSH and child diagnostics can contain user-data or credential-bearing URLs. + raise TransportError('Remote operation failed; inspect the root-owned target status for a redacted diagnosis') + return result + + def fresh_login(self): + self.run('test "$(id -u)" = 0', fresh=True) + + +def protected_handover(ssh, invoke, token, port, client_address): + result = invoke('begin', token, port, client_address, False) + try: + ssh.fresh_login() + if result['status'] == 'committed': + return + if result['status'] != 'existing': + invoke('finalize', token, None, None, True) + ssh.fresh_login() + try: + invoke('commit', token, None, None, True) + except Exception: + try: + state = invoke('status', token, None, None, True) + if state.get('token') != token or state.get('status') not in ('committed', 'existing', 'transition', 'final', 'rolled_back', 'recovery_pending'): + raise TransportError('Unexpected access transition state') + except Exception: + raise HandoverUncertain('Access commit outcome is unknown; verify the administrator and bootstrap endpoints before resuming') from None + if state['status'] != 'committed': + raise + except HandoverUncertain: + raise + except Exception: + try: + invoke('rollback', token, None, None, True) + except Exception: + try: + invoke('rollback', token, None, None, False) + except Exception: + pass # Persistent timer is authoritative if both connections are gone. + raise + + +def initialize(commons, target, user_data, port=None, identity_file=None, progress=print): + if not shutil.which('ssh'): + raise TransportError('OpenSSH ssh is required on the workstation') + source = local_file(user_data, 'User-data') + if source.stat().st_size > 1024 * 1024: + raise TransportError('User-data must be no larger than 1 MiB') + ssh = SSH(target, port, identity_file) + ssh.resolve() + bundle, digest, revision = build_bundle(commons) + run_id = uuid.uuid4().hex + installed = '/usr/local/lib/apex-provisioner/' + digest + stage = None + launched = False + finished = False + access_committed = False + access_uncertain = False + cleanup_policy = '/etc/tmpfiles.d/apex-init-' + run_id + '.conf' + with tempfile.TemporaryDirectory(prefix='apex-init-trust-') as local: + try: + preflight = ssh.run('set -eu; test "$(id -u)" = 0; . /etc/os-release; ' + 'test "$ID" = debian; test "$VERSION_ID" = 13; ' + 'test "$(dpkg --print-architecture)" = amd64; ' + 'command -v systemd-run >/dev/null; ' + 'printf "%s\\n" "${SSH_CONNECTION:-}"; ' + 'cat /etc/ssh/ssh_host_*_key.pub') + lines = preflight.stdout.decode().splitlines() + connection = lines[0].split() if lines else [] + if len(connection) != 4: + # sudo commonly clears SSH_CONNECTION; read it before elevation instead. + raw = ssh.run('printf "%s\\n" "$SSH_CONNECTION"', privileged=False).stdout.decode().split() + connection = raw + if len(connection) != 4: + raise TransportError('Unable to inspect the established SSH connection') + keys = [line for line in lines[1:] if re.fullmatch(r'(ssh-[\w-]+|ecdsa-[\w-]+) [A-Za-z0-9+/=]+(?: .*?)?', line)] + if not keys: + raise TransportError('Unable to obtain public host keys through the verified connection') + trust = Path(local) / 'known_hosts' + trust.write_text(''.join('apex-init-verified-target,[apex-init-verified-target]:2222 ' + ' '.join(key.split()[:2]) + '\n' for key in keys)) + trust.chmod(0o600) + ssh.trust_file = trust + existing = ssh.run(REMOTE_PATH_GUARD + '\napex_ensure_dir /var/lib/apex-init 0700; ' + 'apex_guard_file /var/lib/apex-init/remote.json; ' + 'if test -f /var/lib/apex-init/remote.json; then cat /var/lib/apex-init/remote.json; fi').stdout + if existing: + prior = json.loads(existing) + previous_unit = prior.get('unit', '') + if not re.fullmatch(r'apex-init-[a-f0-9]{32}', previous_unit): + raise TransportError('Existing remote coordinator state requires manual inspection') + active = ssh.run(shlex.join(['systemctl', 'is-active', '--quiet', previous_unit]), check=False) + if active.returncode == 0: + raise TransportError('An enrollment unit is still running: ' + previous_unit + '; wait for it to settle before resuming') + stage = ssh.run(REMOTE_PATH_GUARD + '\napex_guard_dir /var/tmp; mktemp -d /var/tmp/apex-init.XXXXXXXXXX').stdout.decode().strip() + if not re.fullmatch(r'/var/tmp/apex-init\.[A-Za-z0-9]{10}', stage): + raise TransportError('Target returned an invalid staging path') + # Hard reboot can bypass the shell trap; boot-time tmpfiles removes opaque inputs. + cleanup = ('r ' + stage + '/user-data - - - -\n' + + 'r ' + stage + '/heartbeat - - - -\n').encode() + ssh.run(REMOTE_PATH_GUARD + '\napex_ensure_dir /etc/tmpfiles.d 0755; apex_atomic_input ' + cleanup_policy, cleanup) + ssh.run(remote_atomic_input(stage + '/bundle.tar'), bundle) + ssh.run(remote_atomic_input(stage + '/user-data'), source.read_bytes()) + install = remote_bundle_install(stage, installed, digest) + progress('Installing the versioned target runtime and validating enrollment input.') + ssh.run(install) + base = ['python3', '-m', 'engine.provisioning.target'] + options = ['--user-data', stage + '/user-data', '--revision', revision, '--bundle-digest', digest] + # State remains inspectable after disconnect; user-data is removed by the durable unit. + enrollment = shlex.join(base + ['enroll'] + options + ['--heartbeat', stage + '/heartbeat']) + validation = shlex.join(base + ['validate'] + options) + access_preflight = shlex.join(['python3', '-m', 'engine.provisioning.access', 'preflight']) + seed = shlex.join(['bash', installed + '/engine/provisioning/seed.sh']) + wrapper = ('umask 077; cd ' + installed + '; ' + 'trap ' + shlex.quote('rm -f ' + stage + '/user-data ' + stage + '/heartbeat ' + cleanup_policy) + ' EXIT; ' + '{ ' + seed + ' && ' + validation + ' && ' + access_preflight + ' && ' + enrollment + '; } >' + stage + '/runner.log 2>&1; rc=$?; ' + 'printf "%s\\n" "$rc" >' + stage + '/result.new; mv ' + stage + '/result.new ' + stage + '/result; exit "$rc"') + unit = 'apex-init-' + run_id + record = json.dumps({'unit': unit, 'staging': stage, 'revision': revision, 'bundle_digest': digest}).encode() + ssh.run(remote_atomic_input('/var/lib/apex-init/remote.json'), record) + ssh.run('touch ' + stage + '/heartbeat; ' + shlex.join(['systemd-run', '--quiet', '--unit', unit, + '--property=Type=exec', '--property=KillMode=process', '/bin/sh', '-c', wrapper])) + launched = True + progress('Enrollment is running under ' + unit + '; target staging: ' + stage) + while True: + status = ssh.run('if test -f ' + stage + '/result; then cat ' + stage + '/result; ' + 'elif systemctl is-active --quiet ' + unit + '; then touch ' + stage + '/heartbeat; printf running; ' + 'else printf stopped; fi').stdout.decode().strip() + if status != 'running': + finished = True + if status != '0': + raise TransportError('Target provisioning failed; inspect root-owned ' + stage + + '/runner.log and /var/lib/apex-init/enrollment.json') + break + time.sleep(5) + progress('Enrollment verified. Checking protected administrator SSH handover.') + + def access(operation, token, original_port, address, fresh): + if operation == 'status': + result = ssh.run(REMOTE_PATH_GUARD + '\napex_guard_file /var/lib/apex-init/access/state.json; ' + 'cat /var/lib/apex-init/access/state.json', fresh=True) + return json.loads(result.stdout) + args = ['python3', '-m', 'engine.provisioning.access', operation, '--token', token] + if original_port: + args += ['--original-port', str(original_port), '--client-address', address] + result = ssh.run('cd ' + installed + ' && ' + shlex.join(args), fresh=fresh) + return json.loads(result.stdout) + + protected_handover(ssh, access, run_id, int(connection[3]), connection[0]) + access_committed = True + result = ssh.run('cd ' + installed + ' && ' + shlex.join(base + ['complete']), fresh=True) + completed = json.loads(result.stdout) + progress('Initialization and fresh administrator access verified; provisioner ' + revision + ', bundle ' + digest) + repository = completed.get('stages', {}).get('repository', {}).get('result', {}) + if repository.get('commit'): + progress('Repository ' + repository['commit'] + '; commons ' + repository.get('commons_commit', 'unknown')) + return completed + except HandoverUncertain: + access_uncertain = True + raise + finally: + if stage and (not launched or finished): + for fresh in (True, False): + try: + ssh.run('rm -f ' + stage + '/user-data ' + stage + '/heartbeat ' + stage + '/bundle.tar ' + cleanup_policy, fresh=fresh) + break + except Exception: + pass + if launched and not finished: + progress('Coordinator stopped. The current target stage may finish; later stages stop after heartbeat expiry.') + if access_uncertain: + progress('Access commit outcome is unknown; verify which endpoint is active before resuming. Rollback may already be disabled.') + endpoints = [('Administrator candidate', 'adam', 2222), ('Bootstrap candidate', ssh.user, ssh.resolved_port)] + else: + label = 'Safe resume' + ('' if access_committed else ' after any pending rollback') + endpoints = [(label, 'adam' if access_committed else ssh.user, 2222 if access_committed else ssh.resolved_port)] + for label, resume_user, resume_port in endpoints: + resume = './apex init ' + resume_user + '@' + ssh.host + ' --port ' + str(resume_port) + ' --user-data ' + shlex.quote(str(source)) + if ssh.identity: + resume += ' --identity-file ' + shlex.quote(str(ssh.identity)) + progress(label + ': ' + resume) diff --git a/engine/provisioning/userdata.py b/engine/provisioning/userdata.py new file mode 100644 index 0000000..27a0644 --- /dev/null +++ b/engine/provisioning/userdata.py @@ -0,0 +1,29 @@ +"""Target-only decoding; importing the CLI never requires a YAML dependency.""" +from .contract import ContractError, normalize_user_data + + +def decode_user_data(content): + import yaml + + class Loader(yaml.SafeLoader): + def compose_node(self, parent, index): + if self.check_event(yaml.AliasEvent): + raise ContractError('YAML aliases are not supported') + return super().compose_node(parent, index) + + def construct_mapping(self, node, deep=False): + result = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if not isinstance(key, str) or key in result: + raise ContractError('YAML mapping keys must be unique strings') + result[key] = self.construct_object(value_node, deep=deep) + return result + + if len(content) > 1024 * 1024: + raise ContractError('user-data exceeds the supported size') + try: + document = yaml.load(content, Loader=Loader) + except (yaml.YAMLError, UnicodeError, RecursionError): + raise ContractError('invalid or unsupported YAML user-data') from None + return normalize_user_data(document) diff --git a/tests/provision_fakes.py b/tests/provision_fakes.py new file mode 100644 index 0000000..5e7698c --- /dev/null +++ b/tests/provision_fakes.py @@ -0,0 +1,38 @@ +from engine.provisioning.security import SecurityError + + +class SecurityBoundary: + """Isolate firewall/service effects while exercising the durable coordinator.""" + def __init__(self): + self.active = False + self.fail_activation = False + self.fail_restore = False + + def preflight(self): + return {} + + def snapshot(self, plan, port, address): + return {'phase': 'prepared', 'was_active': self.active} + + def activate(self, state, persist): + self.active = True + state['phase'] = 'active' + persist(state) + if self.fail_activation: + raise SecurityError('activation failed') + + def finalize(self, state, persist): + self.verify(state) + state['phase'] = 'final' + persist(state) + + def verify(self, state): + if not self.active: + raise SecurityError('security stopped') + + def restore(self, state, persist): + self.active = state['was_active'] + state['cleanup'] = 'pending' if self.fail_restore else 'complete' + persist(state) + if self.fail_restore: + raise SecurityError('cleanup pending') diff --git a/tests/test_provision_access.py b/tests/test_provision_access.py new file mode 100644 index 0000000..1728874 --- /dev/null +++ b/tests/test_provision_access.py @@ -0,0 +1,513 @@ +import json +from pathlib import Path +import pwd +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from engine.provisioning.access import AccessError, Host, Transition, atomic, main +from provision_fakes import SecurityBoundary + + +TOKEN = 'a' * 32 + + +class FakeHost(Host): + """Replace system service effects; keep snapshots and transition files real.""" + def __init__(self, root): + super().__init__(root) + self.security = SecurityBoundary() + self.boot = 'first-boot' + self.armed = False + self.restored = 0 + self.ports = [22] + self.firewall = False + self.bad_sudo = False + self.fail_final = False + self.existing_final = False + atomic(self.path('/etc/ssh/sshd_config'), b'Port 22\n') + + def boot_id(self): + return self.boot + + def preflight(self): + if self.bad_sudo: + raise AccessError('sudo failed') + + def arm(self): + self.armed = True + + def disarm(self): + self.armed = False + + def firewall_needed(self): + return True + + def open_firewall(self): + self.firewall = True + + def policy(self, ports, final, address): + if final and self.fail_final: + raise AccessError('policy failed') + + def current_final(self, address): + return self.existing_final + + def verify_active(self, ports, final, address): + self.policy(ports, final, address) + if self.ports != sorted(set(ports)): + raise AccessError('Live listener ports changed') + + def install_policy(self, ports, final, address): + self.policy(ports, final, address) + self.ports = sorted(set(ports)) + atomic(self.path('/etc/systemd/system/ssh.service.d/apex-init.conf'), + json.dumps({'ports': self.ports, 'final': final}).encode()) + + def restore(self, state): + self.restored += 1 + self.ports = [state['original_port']] + self.firewall = False + path = self.path('/etc/systemd/system/ssh.service.d/apex-init.conf') + if state['previous'] is None: + path.unlink(missing_ok=True) + else: + import base64 + atomic(path, base64.b64decode(state['previous'])) + + +class AccessTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.host = FakeHost(Path(self.temp.name)) + self.clock = 1000 + self.transition = Transition(self.host, now=lambda: self.clock, security=self.host.security) + + def begin(self): + return self.transition.execute('begin', TOKEN, 22, '192.0.2.10') + + def test_transition_keeps_original_until_final_and_commit_stops_rollback(self): + self.begin() + self.assertEqual(self.host.ports, [22, 2222]) + self.assertTrue(self.host.armed) + self.transition.execute('finalize', TOKEN) + self.assertEqual(self.host.ports, [2222]) + self.assertTrue(self.host.armed) + self.transition.execute('commit', TOKEN) + self.clock += 400 + self.host.boot = 'new-boot' + self.transition.execute('tick') + self.assertEqual(self.host.ports, [2222]) + self.assertEqual(self.host.restored, 0) + self.assertFalse(self.host.armed) + + def test_expiry_restores_original_even_after_final_policy(self): + self.begin() + self.transition.execute('finalize', TOKEN) + self.clock += 301 + self.transition.execute('tick') + self.assertEqual(self.host.ports, [22]) + self.assertFalse(self.host.firewall) + self.assertEqual(self.transition.load()['status'], 'rolled_back') + + def test_reboot_recovers_persisted_transition_with_fresh_runner(self): + self.begin() + self.host.boot = 'second-boot' + fresh = Transition(self.host, now=lambda: 0, security=self.host.security) + fresh.execute('tick') + self.assertEqual(self.host.ports, [22]) + self.assertEqual(self.host.restored, 1) + + def test_watchdog_restores_access_while_enrollment_lock_is_held(self): + from engine.provisioning.state import exclusive_lock + self.begin() + self.clock += 301 + with exclusive_lock(self.host.path('/var/lib/apex-init/init.lock')): + self.transition.execute('tick') + self.assertEqual(self.host.ports, [22]) + self.assertEqual(self.host.restored, 1) + + def test_external_edit_prevents_finalize_and_watchdog_preserves_edit(self): + self.begin() + path = self.host.path('/etc/ssh/sshd_config') + path.write_text('Port 22\nBanner /etc/banner\n') + with self.assertRaises(AccessError): + self.transition.execute('finalize', TOKEN) + self.assertTrue(self.host.armed) + self.clock += 301 + self.transition.execute('tick') + self.assertIn('Banner', path.read_text()) + + def test_invalid_sudo_never_changes_listeners(self): + self.host.bad_sudo = True + with self.assertRaises(AccessError): + self.begin() + self.assertEqual(self.host.ports, [22]) + self.assertFalse(self.host.armed) + + def test_failed_final_policy_restores_original(self): + self.begin() + self.host.fail_final = True + with self.assertRaises(AccessError): + self.transition.execute('finalize', TOKEN) + self.assertEqual(self.host.ports, [22]) + self.assertFalse(self.host.armed) + + def test_other_coordinator_cannot_commit(self): + self.begin() + self.transition.execute('finalize', TOKEN) + with self.assertRaises(AccessError): + self.transition.execute('commit', 'b' * 32) + self.assertTrue(self.host.armed) + + def test_committed_retry_does_not_reopen_root_listener(self): + self.begin() + self.transition.execute('finalize', TOKEN) + self.transition.execute('commit', TOKEN) + result = self.transition.execute('begin', 'b' * 32, 2222, '192.0.2.10') + self.assertEqual(result['status'], 'committed') + self.assertEqual(self.host.ports, [2222]) + self.assertFalse(self.host.armed) + + def test_committed_retry_rejects_live_listener_drift(self): + self.begin() + self.transition.execute('finalize', TOKEN) + self.transition.execute('commit', TOKEN) + self.host.ports = [22, 2222] + with self.assertRaises(AccessError): + self.transition.execute('begin', 'b' * 32, 2222, '192.0.2.10') + + def test_preexisting_final_policy_is_preserved_during_security_activation(self): + self.host.existing_final = True + self.host.ports = [2222] + result = self.transition.execute('begin', TOKEN, 2222, '192.0.2.10') + self.assertEqual(result['status'], 'transition') + self.assertTrue(self.host.armed) + self.assertFalse(self.host.firewall) + self.transition.execute('finalize', TOKEN) + self.transition.execute('commit', TOKEN) + self.assertEqual(self.transition.load()['status'], 'committed') + self.assertFalse(self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf').exists()) + + def test_reject_symlink_state_ancestor(self): + outside = self.host.path('/outside') + outside.mkdir() + self.host.path('/var').mkdir() + self.host.path('/var/lib').symlink_to(outside) + with self.assertRaises(AccessError): + self.begin() + self.assertEqual(list(outside.iterdir()), []) + + def test_policy_rejects_additive_ports(self): + def runner(args, **kwargs): + text = 'port 22\nport 2222\n' if '-T' in args else '' + return subprocess.CompletedProcess(args, 0, text, '') + with self.assertRaises(AccessError): + Host(self.host.root, runner).policy([2222], True, '192.0.2.10') + + def test_atomic_snapshot_does_not_follow_predictable_temporary_link(self): + destination = self.host.root / 'snapshot' + victim = self.host.root / 'victim' + victim.write_bytes(b'preserved') + destination.with_name('snapshot.new').symlink_to(victim) + atomic(destination, b'new snapshot') + self.assertEqual(destination.read_bytes(), b'new snapshot') + self.assertEqual(victim.read_bytes(), b'preserved') + + +class LivePolicyTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.arguments = ['/usr/sbin/sshd', '-D', '-p', '2222'] + self.running_arguments = self.arguments[:] + self.effective = ('port 2222\npermitrootlogin no\npasswordauthentication no\n' + 'kbdinteractiveauthentication no\npubkeyauthentication yes\n' + 'authenticationmethods publickey\nallowusers adam\n' + 'x11forwarding no\npermitemptypasswords no\n') + self.listeners = 'LISTEN 0 128 0.0.0.0:2222 0.0.0.0:* users:(("sshd",pid=755,fd=6))\n' + self.calls = [] + self.host = Host(self.root, self.runner) + atomic(self.host.path('/etc/ssh/sshd_config'), b'# fixture\n') + process = self.root / 'proc/755' + process.mkdir(parents=True) + (process / 'exe').symlink_to('/usr/sbin/sshd') + self.cmdline() + + def cmdline(self, title=False): + data = ('sshd: ' + ' '.join(self.running_arguments) + ' [listener] 0 of 10-100 startups\0' + if title else '\0'.join(self.running_arguments) + '\0') + (self.root / 'proc/755/cmdline').write_bytes(data.encode()) + + def runner(self, args, **kwargs): + self.calls.append(args) + if args[:2] in (['systemctl', 'is-active'], ['systemctl', 'is-enabled']) and args[-1] == 'ssh.socket': + return subprocess.CompletedProcess(args, 3, '', '') + if 'ExecStart' in args: + output = '{ path=/usr/sbin/sshd ; argv[]=' + ' '.join(self.arguments) + ' ; ignore_errors=no ; }' + elif 'MainPID' in args: + output = '755\n' + elif 'KillMode' in args: + output = 'process\n' + elif args[0] == 'ss': + output = self.listeners + elif '-T' in args: + output = self.effective + else: + output = '' + return subprocess.CompletedProcess(args, 0, output, '') + + def test_verifies_exact_live_arguments_without_inventing_auth_options(self): + self.cmdline(title=True) + self.host.verify_active([2222], True, '192.0.2.1') + effective_call = next(args for args in self.calls if '-T' in args) + self.assertEqual(effective_call[:3], ['/usr/sbin/sshd', '-p', '2222']) + self.assertNotIn('PermitRootLogin=no', effective_call) + + def test_password_or_root_enabled_on_live_daemon_is_rejected(self): + for setting in ('passwordauthentication', 'permitrootlogin'): + with self.subTest(setting=setting): + previous = self.effective + self.effective = previous.replace(setting + ' no', setting + ' yes') + with self.assertRaises(AccessError): + self.host.verify_active([2222], True, '192.0.2.1') + self.effective = previous + + def test_inherited_final_restrictions_cannot_silently_allow_other_accounts(self): + for previous, conflicting in (('allowusers adam', 'allowusers adam\nallowusers provider'), + ('x11forwarding no', 'x11forwarding yes'), + ('permitemptypasswords no', 'permitemptypasswords yes')): + with self.subTest(conflicting=conflicting): + original = self.effective + self.effective = original.replace(previous, conflicting) + with self.assertRaises(AccessError): + self.host.verify_active([2222], True, '192.0.2.1') + self.effective = original + + def test_bootstrap_prerequisites_do_not_require_adam_or_sudo_or_write_state(self): + self.arguments = self.running_arguments = ['/usr/sbin/sshd', '-D'] + self.cmdline() + def unseeded(args, **kwargs): + if args[0] in ('visudo', 'runuser', 'sudo'): + raise AssertionError('administrator tools do not exist before baseline') + return self.runner(args, **kwargs) + self.host.runner = unseeded + before = sorted(self.root.rglob('*')) + with patch('engine.provisioning.access.Host', return_value=self.host), \ + patch('engine.provisioning.access.os.geteuid', return_value=0): + self.assertEqual(main(['preflight']), 0) + self.assertEqual(sorted(self.root.rglob('*')), before) + self.assertIn(['/usr/sbin/sshd', '-t'], self.calls) + self.assertTrue(any('MainPID' in args for args in self.calls)) + + def test_bootstrap_prerequisites_reject_socket_and_live_invocation_drift(self): + self.arguments = self.running_arguments = ['/usr/sbin/sshd', '-D'] + self.cmdline() + for failure in ('socket', 'invocation'): + with self.subTest(failure=failure): + def incompatible(args, **kwargs): + if failure == 'socket' and args == ['systemctl', 'is-enabled', '--quiet', 'ssh.socket']: + return subprocess.CompletedProcess(args, 0, '', '') + return self.runner(args, **kwargs) + self.host.runner = incompatible + self.running_arguments = ['/usr/sbin/sshd', '-D', '-p', '22'] if failure == 'invocation' else self.arguments + self.cmdline() + with self.assertRaises(AccessError): + self.host.bootstrap_preflight() + + def test_later_preflight_still_checks_administrator_sudo(self): + self.arguments = self.running_arguments = ['/usr/sbin/sshd', '-D'] + self.cmdline() + def bad_sudo(args, **kwargs): + if args[:3] == ['runuser', '-u', 'adam']: + return subprocess.CompletedProcess(args, 1, '', 'sudo denied') + return self.runner(args, **kwargs) + self.host.runner = bad_sudo + with self.assertRaises(AccessError): + self.host.preflight() + + def test_bootstrap_follows_recursive_relative_and_quoted_include_paths(self): + self.arguments = self.running_arguments = ['/usr/sbin/sshd', '-D'] + self.cmdline() + main_config = self.host.path('/etc/ssh/sshd_config') + first = self.host.path('/etc/ssh/sshd_config.d/10-first.conf') + second = self.host.path('/etc/ssh/provider # policy.conf') + atomic(main_config, b'Include sshd_config.*/[0-9]*.conf /etc/ssh/missing/*.conf\n') + atomic(first, b'iNcLuDe = "provider # policy.conf" # relative to /etc/ssh\n') + atomic(second, b'# Match Address 203.0.113.0/24\nBanner "/etc/Match banner"\n') + self.host.bootstrap_preflight() + second.write_text('"mAtCh" Address 203.0.113.0/24\n PasswordAuthentication yes\n') + before = {path: path.read_bytes() for path in (main_config, first, second)} + with self.assertRaisesRegex(AccessError, 'Match.*manual reconciliation'): + self.host.bootstrap_preflight() + self.assertEqual({path: path.read_bytes() for path in before}, before) + + def test_bootstrap_rejects_recursive_or_oversized_config_without_mutation(self): + self.arguments = self.running_arguments = ['/usr/sbin/sshd', '-D'] + self.cmdline() + configuration = self.host.path('/etc/ssh/sshd_config') + for content in ('Include sshd_config\n', '#' + 'x' * (1024 * 1024 + 1)): + with self.subTest(recursive=content.startswith('Include')): + configuration.write_text(content) + with self.assertRaisesRegex(AccessError, 'manual reconciliation'): + self.host.bootstrap_preflight() + self.assertEqual(configuration.read_text(), content) + + def test_extended_include_glob_tokens_are_rejected_even_without_matches(self): + self.arguments = self.running_arguments = ['/usr/sbin/sshd', '-D'] + self.cmdline() + configuration = self.host.path('/etc/ssh/sshd_config') + for pattern in ('provider[a[:digit:]].conf', 'provider[a[.x.]].conf', + 'provider[a[=x=]].conf', 'provider[:digit:].conf'): + with self.subTest(pattern=pattern): + content = 'Include ' + pattern + '\n' + configuration.write_text(content) + with self.assertRaisesRegex(AccessError, 'glob syntax.*manual reconciliation'): + self.host.bootstrap_preflight() + self.assertEqual(configuration.read_text(), content) + + def test_unrestarted_daemon_with_old_arguments_is_rejected(self): + self.running_arguments = ['/usr/sbin/sshd', '-D', '-p', '22'] + self.cmdline() + with self.assertRaises(AccessError): + self.host.verify_active([2222], True, '192.0.2.1') + + def test_extra_live_listener_port_is_rejected(self): + self.listeners += 'LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=755,fd=7))\n' + with self.assertRaises(AccessError): + self.host.verify_active([2222], True, '192.0.2.1') + + def test_additional_sshd_service_listener_is_rejected(self): + self.listeners += 'LISTEN 0 128 0.0.0.0:22 0.0.0.0:* users:(("sshd",pid=999,fd=7))\n' + with self.assertRaises(AccessError): + self.host.verify_active([2222], True, '192.0.2.1') + + def test_runtime_override_changes_fingerprint(self): + before = self.host.fingerprint() + atomic(self.host.path('/run/systemd/system/ssh.service.d/override.conf'), b'[Service]\nExecStart=/usr/sbin/sshd -D\n') + self.assertNotEqual(before, self.host.fingerprint()) + + def test_managed_file_does_not_exempt_drifted_service_from_preflight(self): + atomic(self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf'), + b'[Service]\nExecStart=\nExecStart=/usr/sbin/sshd -p 2222 -o PermitRootLogin=no -D\n') + with self.assertRaisesRegex(AccessError, 'differs from its managed invocation'): + self.host.preflight() + + +@unittest.skipUnless(shutil.which('sshd') and shutil.which('ssh-keygen'), 'OpenSSH tools unavailable') +class OpenSSHPolicyTests(unittest.TestCase): + """Evaluate the managed options with a real daemon without opening sockets.""" + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.key = self.root / 'host-key' + self.config = self.root / 'etc/ssh/sshd_config' + self.config.parent.mkdir(parents=True) + self.config.write_text('PasswordAuthentication yes\nPermitRootLogin yes\n' + 'X11Forwarding yes\nPermitEmptyPasswords yes\n') + subprocess.run(['ssh-keygen', '-q', '-t', 'ed25519', '-N', '', '-f', str(self.key)], + check=True, capture_output=True, timeout=30) + self.host = Host(self.root, self.runner) + original_path = self.host.path + self.host.path = lambda path: Path(path) if Path(path).is_relative_to(self.root) else original_path(path) + + def runner(self, args, **kwargs): + if args[0] != '/usr/sbin/sshd': + code = 3 if args[-1] == 'ssh.socket' else 0 + output = 'process\n' if 'KillMode' in args else '' + return subprocess.CompletedProcess(args, code, output, '') + return subprocess.run([shutil.which('sshd'), '-f', str(self.config), '-h', str(self.key), *args[1:]], + **kwargs) + + def effective(self, arguments, user): + result = self.runner(arguments + ['-T', '-C', 'user=' + user + ',host=localhost,addr=192.0.2.1'], + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout.splitlines() + + def test_transition_preserves_provider_access_and_only_final_policy_excludes_it(self): + transition = self.host.policy([22, 2222], False, '192.0.2.1') + transitional = self.effective(transition, 'provider') + self.assertFalse(any(line.startswith('allowusers ') for line in transitional)) + self.assertIn('passwordauthentication yes', transitional) + final = self.host.policy([2222], True, '192.0.2.1') + for user in ('adam', 'provider'): + with self.subTest(user=user): + effective = self.effective(final, user) + allowed = [line.removeprefix('allowusers ') for line in effective if line.startswith('allowusers ')] + self.assertEqual(allowed, ['adam']) + self.assertNotIn('provider', allowed) + self.assertIn('x11forwarding no', effective) + self.assertIn('permitemptypasswords no', effective) + + def test_additive_inherited_allowusers_stops_finalization(self): + with self.config.open('a') as stream: + stream.write('AllowUsers provider\n') + self.host.policy([22, 2222], False, '192.0.2.1') + with self.assertRaises(AccessError): + self.host.policy([2222], True, '192.0.2.1') + + def test_provider_account_match_block_cannot_bypass_final_restrictions(self): + provider = pwd.struct_passwd(('provider', 'x', 1001, 1001, '', '/home/provider', '/bin/bash')) + for setting in ('AllowUsers provider', 'X11Forwarding yes', 'PermitEmptyPasswords yes'): + with self.subTest(setting=setting): + self.config.write_text('Match User provider\n ' + setting + '\n') + with patch('pwd.getpwall', return_value=[provider]), self.assertRaises(AccessError): + self.host.policy([2222], True, '192.0.2.1') + + def test_bootstrap_rejects_match_address_in_main_or_recursively_included_file(self): + conditional = ('\n AllowUsers root provider\n PasswordAuthentication yes\n' + ' PermitRootLogin yes\n AuthenticationMethods any\n' + ' X11Forwarding yes\n PermitEmptyPasswords yes\n') + first = self.root / 'first # include.conf' + second = self.root / 'second include.conf' + for directive, included in (('Match Address', False), ('mAtCh=Address', False), + ('"Match" Address', False), ('"mAtCh" Address', True)): + with self.subTest(directive=directive, included=included): + payload = directive + ' 203.0.113.0/24' + conditional + if included: + second.write_text(payload) + first.write_text('iNcLuDe "' + str(second) + '"\n') + self.config.write_text('"Include" "' + str(first) + '" # provider policy\n') + else: + self.config.write_text(payload) + arguments = ['/usr/sbin/sshd', '-T', '-p', '2222', '-o', 'AllowUsers=adam', + '-o', 'PermitRootLogin=no', '-o', 'PasswordAuthentication=no', + '-o', 'AuthenticationMethods=publickey', '-o', 'X11Forwarding=no', + '-o', 'PermitEmptyPasswords=no'] + other_address = self.runner(arguments + ['-C', 'user=root,host=localhost,addr=203.0.113.25'], + capture_output=True, text=True, timeout=30) + self.assertEqual(other_address.returncode, 0, other_address.stderr) + self.assertIn('permitrootlogin yes', other_address.stdout.splitlines()) + self.assertIn('passwordauthentication yes', other_address.stdout.splitlines()) + before = self.config.read_bytes() + with patch.object(self.host, 'service_arguments', return_value=['/usr/sbin/sshd', '-D']), \ + patch.object(self.host, 'live_arguments'), \ + self.assertRaisesRegex(AccessError, 'Match.*manual reconciliation'): + self.host.bootstrap_preflight() + self.assertEqual(self.config.read_bytes(), before) + + def test_posix_class_include_cannot_hide_a_match_from_preflight(self): + included = self.root / 'provider1.conf' + included.write_text('Match Address 203.0.113.0/24\n PermitRootLogin yes\n') + for pattern in ('provider[[:digit:]].conf', 'provider[a[:digit:]].conf'): + with self.subTest(pattern=pattern): + self.config.write_text('Include ' + str(self.root / pattern) + '\n') + effective = self.runner(['/usr/sbin/sshd', '-T', '-o', 'PermitRootLogin=no', + '-C', 'user=root,host=localhost,addr=203.0.113.25'], + capture_output=True, text=True, timeout=30) + self.assertEqual(effective.returncode, 0, effective.stderr) + self.assertIn('permitrootlogin yes', effective.stdout.splitlines()) + with patch.object(self.host, 'service_arguments', return_value=['/usr/sbin/sshd', '-D']), \ + patch.object(self.host, 'live_arguments'), \ + self.assertRaisesRegex(AccessError, 'manual reconciliation'): + self.host.bootstrap_preflight() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_provision_baseline.py b/tests/test_provision_baseline.py new file mode 100644 index 0000000..c3e0b1f --- /dev/null +++ b/tests/test_provision_baseline.py @@ -0,0 +1,338 @@ +"""Exercise baseline decisions with a temporary host filesystem and fake commands.""" +import hashlib +import io +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from engine.provisioning import baseline + + +class Host: + def __init__(self, root): + self.root = root + self.commands = [] + self.installed = {} + self.partial = {} + self.active = set() + self.enabled = set() + self.masked = set() + self.fail_install = False + self.plan = '' + self.zfs_available = True + self.docker_root = '/srv/docker' + self.path('/etc/os-release').parent.mkdir(parents=True) + self.path('/etc/os-release').write_text('ID=debian\nVERSION_ID="13"\n') + self.path('/etc/debian_version').touch() + self.path('/usr/sbin').mkdir(parents=True) + + def path(self, path): + path = Path(path) + if str(path).startswith(str(self.root)): + return path + return self.root / str(path).lstrip('/') + + def command(self, args, **kwargs): + self.commands.append(args) + code, output = 0, '' + if args[:2] == ['dpkg', '--print-architecture']: + output = 'amd64\n' + elif args[0] == 'dpkg-query': + output = ''.join(f'{name}\tinstall ok installed\t{version}\n' for name, version in self.installed.items()) + output += ''.join(f'{name}\t{status}\t1.0\n' for name, status in self.partial.items()) + elif args == ['uname', '-r']: + output = '6.12.0-amd64\n' + elif args[0] == 'apt-get' and 'install' in args: + if '--simulate' in args: + output = self.plan + elif self.fail_install: + code = 100 + else: + start = args.index('--no-remove') + 1 + for name in args[start:]: + self.installed[name] = '1.0' + self.partial.pop(name, None) + elif args[0] == 'systemctl': + service = args[-1] + operation = args[1] + if operation == 'is-active': + code = 0 if service in self.active else 3 + elif operation == 'is-enabled': + code = 0 if service in self.enabled else 1 + output = 'masked\n' if service in self.masked else '' + elif operation == 'start': + conditions = [] + for dropin in self.path(f'/etc/systemd/system/{service}.service.d').glob('*.conf'): + conditions.extend(line.partition('=')[2] for line in dropin.read_text().splitlines() + if line.startswith('ConditionPathExists=')) + if service not in self.masked and all(self.path(path).exists() for path in conditions): + self.active.add(service) + elif operation == 'enable': + self.enabled.add(service) + elif operation == 'disable': + self.enabled.discard(service) + elif operation == 'mask': + self.masked.add(service) + unit = self.path(f'/etc/systemd/system/{service}.service') + unit.parent.mkdir(parents=True, exist_ok=True) + unit.symlink_to('/dev/null') + elif operation == 'unmask': + self.masked.discard(service) + unit = self.path(f'/etc/systemd/system/{service}.service') + if unit.is_symlink(): + unit.unlink() + elif args[:2] == ['docker', 'info']: + output = self.docker_root + '\n' + elif args[:3] == ['docker', 'compose', 'version']: + output = '2.39.2\n' + elif args == ['modprobe', 'zfs']: + code = 0 if self.zfs_available else 1 + elif args == ['zfs', 'version']: + output = 'zfs-2.3.2\nzfs-kmod-2.3.2\n' + return subprocess.CompletedProcess(args, code, output, 'installation failed' if code else '') + + +class BaselineTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.host = Host(Path(self.temp.name).resolve()) + self.helper = b'#!/bin/bash\necho pinned test helper\n' + patches = [ + patch.object(baseline, 'Path', self.host.path), + patch.object(baseline, 'POLICY', self.host.path('/usr/sbin/policy-rc.d')), + patch.object(baseline, 'POLICY_STATE', self.host.path('/usr/sbin/.apex-policy-rc.d')), + patch.object(baseline, 'SECURITY_STATE', self.host.path('/var/lib/apex/baseline-security.json')), + patch.object(baseline.os, 'geteuid', return_value=Path(self.temp.name).stat().st_uid), + patch.object(baseline.subprocess, 'run', side_effect=self.host.command), + patch.object(baseline.urllib.request, 'urlopen', side_effect=self.download), + patch.object(baseline, 'HELPER_SHA256', hashlib.sha256(self.helper).hexdigest()), + ] + for item in patches: + item.start() + self.addCleanup(item.stop) + + def download(self, url, **kwargs): + if url == baseline.HELPER_URL: + return io.BytesIO(self.helper) + return io.BytesIO(b'-----BEGIN PGP PUBLIC KEY BLOCK-----\nfixture\n') + + def apply(self): + # The filesystem remains owned by the test uid; only root preflight is isolated. + with patch.object(baseline, 'preflight'): + return baseline.apply_baseline() + + def test_package_database_drives_installation_and_repeat_does_not_upgrade(self): + self.host.installed['curl'] = '7.0' + self.host.partial['rsyslog'] = 'deinstall ok config-files' + self.assertEqual(baseline.install_missing(['curl', 'rsyslog']), ['rsyslog']) + self.assertEqual(self.host.installed['curl'], '7.0') + self.assertEqual(self.host.installed['rsyslog'], '1.0') + count = len(self.host.commands) + self.assertEqual(baseline.install_missing(['curl', 'rsyslog']), []) + self.assertFalse(any(command[0] == 'apt-get' for command in self.host.commands[count:])) + + def test_dependency_upgrade_is_rejected_before_install(self): + self.host.plan = 'Inst libc6 [2.0] (3.0 Debian:13)\n' + with self.assertRaisesRegex(baseline.BaselineError, 'upgrade/removal'): + baseline.install_missing(['git']) + self.assertNotIn('git', self.host.installed) + + def test_held_installed_packages_are_preserved(self): + self.host.partial['git'] = 'hold ok installed' + self.assertEqual(baseline.install_missing(['git']), []) + self.assertFalse(any(command[0] == 'apt-get' for command in self.host.commands)) + + def test_conflicting_docker_config_is_rejected_without_host_mutation(self): + path = self.host.path('/etc/docker/daemon.json') + path.parent.mkdir(parents=True) + path.write_text('{"data-root": "/data/docker"}') + before = sorted(str(path.relative_to(self.host.root)) for path in self.host.root.rglob('*')) + with self.assertRaisesRegex(baseline.BaselineError, 'Conflicting Docker'): + self.apply() + self.assertEqual(before, sorted(str(path.relative_to(self.host.root)) for path in self.host.root.rglob('*'))) + self.assertFalse(any(command[0] in ('apt-get', 'systemctl') for command in self.host.commands)) + + def test_existing_docker_data_is_preserved(self): + data = self.host.path('/var/lib/docker/volumes/user-data') + data.parent.mkdir(parents=True) + data.write_text('valuable') + with self.assertRaisesRegex(baseline.BaselineError, 'Existing /var/lib/docker'): + self.apply() + self.assertEqual(data.read_text(), 'valuable') + + def test_conflicting_distribution_docker_package_is_never_removed(self): + self.host.installed['docker.io'] = '26.1.5' + with self.assertRaisesRegex(baseline.BaselineError, 'Conflicting Docker packages'): + self.apply() + self.assertEqual(self.host.installed['docker.io'], '26.1.5') + self.assertFalse(any(command[0] == 'apt-get' for command in self.host.commands)) + + def test_unmanaged_destination_data_is_not_adopted(self): + data = self.host.path('/srv/docker/private-data') + data.parent.mkdir(parents=True) + data.write_text('preserve me') + with self.assertRaisesRegex(baseline.BaselineError, 'Unmanaged /srv/docker'): + self.apply() + self.assertEqual(data.read_text(), 'preserve me') + + def test_package_failure_restores_existing_service_policy(self): + policy = self.host.path('/usr/sbin/policy-rc.d') + policy.write_text('#!/bin/sh\nexit 42\n') + policy.chmod(0o751) + self.host.fail_install = True + with self.assertRaises(baseline.BaselineError): + self.apply() + self.assertEqual(policy.read_text(), '#!/bin/sh\nexit 42\n') + self.assertEqual(policy.stat().st_mode & 0o777, 0o751) + self.assertFalse(self.host.path('/usr/sbin/.apex-policy-rc.d').exists()) + self.assertFalse(self.host.masked) + + def test_service_policy_symlink_is_restored_after_failure(self): + policy = self.host.path('/usr/sbin/policy-rc.d') + original = self.host.path('/usr/sbin/provider-policy') + original.write_text('provider policy') + policy.symlink_to(original) + with self.assertRaisesRegex(RuntimeError, 'boom'): + with baseline.service_start_guard(): + self.assertFalse(policy.is_symlink()) + raise RuntimeError('boom') + self.assertTrue(policy.is_symlink()) + self.assertEqual(original.read_text(), 'provider policy') + + def test_interrupted_guard_is_recovered_before_next_attempt(self): + state = self.host.path('/usr/sbin/.apex-policy-rc.d') + state.mkdir() + (state / 'original').write_text('original policy') + (state / 'ready').touch() + self.host.path('/usr/sbin/policy-rc.d').write_text('#!/bin/sh\nexit 101\n') + with baseline.service_start_guard(): + pass + self.assertEqual(self.host.path('/usr/sbin/policy-rc.d').read_text(), 'original policy') + + def test_external_policy_change_is_not_overwritten(self): + policy = self.host.path('/usr/sbin/policy-rc.d') + with self.assertRaisesRegex(baseline.BaselineError, 'changed outside APEX'): + with baseline.service_start_guard(): + policy.write_text('external policy') + self.assertEqual(policy.read_text(), 'external policy') + + def test_baseline_reports_capabilities_and_repeats_without_restarts(self): + result = self.apply() + self.assertEqual(result['capabilities']['docker_data_root'], '/srv/docker') + self.assertEqual(result['capabilities']['compose'], '2.39.2') + self.assertIn('zfs-kmod-2.3.2', result['capabilities']['zfs']) + self.assertEqual(result['packages']['zfs-dkms'], '1.0') + self.assertEqual(self.host.path('/etc/apt/apt.conf.d/99-apex').read_text(), 'APT::Keep-Downloaded-Packages "0";\n') + self.assertFalse(self.host.path('/usr/sbin/policy-rc.d').exists()) + self.assertFalse(self.host.masked) + self.assertNotIn('ufw', self.host.active) + self.assertNotIn('crowdsec-firewall-bouncer', self.host.enabled) + self.assertFalse(any(command[0] in ('ufw', '/usr/local/bin/ufw-docker') for command in self.host.commands)) + count = len(self.host.commands) + again = self.apply() + self.assertEqual(again, result) + repeated = self.host.commands[count:] + self.assertFalse(any(command[0] == 'apt-get' or command[:2] in + (['systemctl', 'restart'], ['systemctl', 'start']) for command in repeated)) + + def test_zfs_missing_for_running_kernel_is_an_explicit_incomplete_result(self): + self.host.zfs_available = False + with self.assertRaisesRegex(baseline.BaselineError, 'reboot.*retry'): + self.apply() + + def test_narrow_base_policy_preserves_scope_and_shared_configuration(self): + with patch.object(baseline, 'preflight'): + result = baseline.apply_base_policy() + self.assertEqual(set(result['packages']), {'curl', 'wget', 'git', 'rsyslog', 'ufw'}) + self.assertFalse(self.host.path('/etc/docker/daemon.json').exists()) + self.assertEqual(self.host.path('/etc/systemd/journald.conf.d/99-apex.conf').read_text(), + '[Journal]\nSystemMaxUse=200M\n') + self.assertIn(['/usr/local/bin/ufw-docker', 'install'], self.host.commands) + + def test_unverified_download_is_not_installed_or_executed(self): + with patch.object(baseline.urllib.request, 'urlopen', return_value=io.BytesIO(b'tampered')): + with self.assertRaisesRegex(baseline.BaselineError, 'checksum mismatch'): + baseline.configure_common() + self.assertFalse(self.host.path('/usr/local/bin/ufw-docker').exists()) + self.assertFalse(any(command[0] == '/usr/local/bin/ufw-docker' for command in self.host.commands)) + + def test_recovery_does_not_remove_preexisting_service_masks(self): + self.host.masked.add('ufw') + with baseline.security_service_guard(['ufw']): + self.host.installed['ufw'] = '0.36' + self.assertIn('ufw', self.host.masked) + + def test_package_owned_etc_service_file_can_be_unpacked_under_guard(self): + unit = self.host.path('/etc/systemd/system/crowdsec-firewall-bouncer.service') + with baseline.security_service_guard(['crowdsec-firewall-bouncer-iptables']): + self.assertFalse(unit.is_symlink(), 'dpkg must be able to install its own conffile') + unit.parent.mkdir(parents=True, exist_ok=True) + unit.write_text('[Service]\nExecStart=/usr/bin/crowdsec-firewall-bouncer\n') + self.host.installed['crowdsec-firewall-bouncer-iptables'] = '0.0.36' + self.host.enabled.add('crowdsec-firewall-bouncer') + baseline.run(['systemctl', 'start', 'crowdsec-firewall-bouncer']) + self.assertNotIn('crowdsec-firewall-bouncer', self.host.active) + self.assertTrue(unit.is_file()) + self.assertIn('ExecStart=/usr/bin/crowdsec-firewall-bouncer', unit.read_text()) + self.assertNotIn('crowdsec-firewall-bouncer', self.host.enabled) + baseline.run(['systemctl', 'start', 'crowdsec-firewall-bouncer']) + self.assertIn('crowdsec-firewall-bouncer', self.host.active) + + def test_disable_failure_retains_guard_and_retry_recovers_it(self): + original_command = self.host.command + def fail_disable(args, **kwargs): + if args == ['systemctl', 'disable', 'crowdsec-firewall-bouncer']: + return subprocess.CompletedProcess(args, 1, '', '') + return original_command(args, **kwargs) + with patch.object(baseline.subprocess, 'run', side_effect=fail_disable): + with self.assertRaises(baseline.BaselineError): + with baseline.security_service_guard(['crowdsec-firewall-bouncer-iptables']): + self.host.installed['crowdsec-firewall-bouncer-iptables'] = '0.0.36' + self.host.enabled.add('crowdsec-firewall-bouncer') + baseline.run(['systemctl', 'start', 'crowdsec-firewall-bouncer']) + self.assertNotIn('crowdsec-firewall-bouncer', self.host.active) + with baseline.security_service_guard(['crowdsec-firewall-bouncer-iptables']): + pass + self.assertNotIn('crowdsec-firewall-bouncer', self.host.enabled) + baseline.run(['systemctl', 'start', 'crowdsec-firewall-bouncer']) + self.assertIn('crowdsec-firewall-bouncer', self.host.active) + + def test_generic_package_failure_does_not_suggest_zfs_reboot(self): + original_command = self.host.command + def fail_security_packages(args, **kwargs): + if args[0] == 'apt-get' and 'crowdsec-firewall-bouncer-iptables' in args and '--simulate' not in args: + return subprocess.CompletedProcess(args, 100, '', 'secret source diagnostic') + return original_command(args, **kwargs) + with patch.object(baseline.subprocess, 'run', side_effect=fail_security_packages): + with self.assertRaises(baseline.BaselineError) as failure: + self.apply() + self.assertNotIn('reboot', str(failure.exception)) + self.assertNotIn('secret source diagnostic', str(failure.exception)) + + def test_unsupported_os_fails_before_package_commands(self): + self.host.path('/etc/os-release').write_text('ID=ubuntu\nVERSION_ID=24.04\n') + with patch.object(baseline.os, 'geteuid', return_value=0): + with self.assertRaisesRegex(baseline.BaselineError, 'Debian 13 amd64'): + baseline.apply_baseline() + self.assertEqual(self.host.commands, []) + + def test_non_root_fails_before_any_command(self): + with patch.object(baseline.os, 'geteuid', return_value=1000): + with self.assertRaisesRegex(baseline.BaselineError, 'root'): + baseline.apply_baseline() + self.assertEqual(self.host.commands, []) + + def test_unsupported_architecture_fails_before_apt_mutation(self): + with patch.object(baseline.os, 'geteuid', return_value=0), patch.object( + baseline.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0, 'arm64\n', '')) as command: + with self.assertRaisesRegex(baseline.BaselineError, 'Debian 13 amd64'): + baseline.apply_baseline() + self.assertEqual([call.args[0] for call in command.call_args_list], [['dpkg', '--print-architecture']]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_provision_contract.py b/tests/test_provision_contract.py new file mode 100644 index 0000000..6036daf --- /dev/null +++ b/tests/test_provision_contract.py @@ -0,0 +1,230 @@ +import copy +import unittest + +from engine.provisioning.contract import ContractError, normalize_user_data, parse_bootstrap_env + + +def example(): + return { + 'preserve_hostname': False, + 'hostname': 'metis.a1.example.com', + 'users': [{ + 'name': 'adam', + 'sudo': 'ALL=(ALL) NOPASSWD:ALL', + 'groups': ['sudo', 'docker'], + 'shell': '/bin/bash', + 'ssh_authorized_keys': ['ssh-ed25519 PUBLIC_TEST_KEY operator'], + }], + 'write_files': [{ + 'path': '/etc/apex/bootstrap.env', + 'owner': 'root:root', + 'permissions': '0600', + 'content': 'APEX_REPO_NAME="metis.example.com"\n' + 'APEX_REPO_URL="https://example.com/team/metis.git"\n', + }], + 'runcmd': ['/usr/local/bin/apex-bootstrap'], + } + + +def add_env(doc, text): + doc['write_files'][0]['content'] += text + return doc + + +class TestBootstrapAssignments(unittest.TestCase): + def test_literal_quotes_comments_and_empty_values(self): + values = parse_bootstrap_env( + '# enrollment\nAPEX_REPO_NAME=metis.example.com\n' + "APEX_REPO_REF='release-1'\nAPEX_TIER1_DEVICE=\n") + self.assertEqual(values['APEX_REPO_REF'], 'release-1') + self.assertEqual(values['APEX_TIER1_DEVICE'], '') + + def test_rejects_shell_forms_without_repeating_input(self): + payloads = [ + 'APEX_REPO_NAME=$(touch /tmp/SECRET_SENTINEL)', + 'APEX_REPO_NAME="${SECRET_SENTINEL}"', + 'APEX_REPO_NAME=`SECRET_SENTINEL`', + 'APEX_REPO_NAME=x;SECRET_SENTINEL', + 'export APEX_REPO_NAME=SECRET_SENTINEL', + ] + for payload in payloads: + with self.subTest(payload=payload): + with self.assertRaises(ContractError) as caught: + parse_bootstrap_env(payload) + self.assertNotIn('SECRET_SENTINEL', str(caught.exception)) + + def test_rejects_duplicate_and_unknown_assignments(self): + for text in ('APEX_REPO_NAME=a\nAPEX_REPO_NAME=b\n', 'OTHER_SECRET=value\n'): + with self.subTest(text=text): + with self.assertRaises(ContractError): + parse_bootstrap_env(text) + + def test_rejects_unquoted_parentheses_and_inline_comments(self): + for value in ('(main)', 'main(', 'main)', ' #comment', 'main #comment'): + with self.subTest(value=value): + with self.assertRaises(ContractError): + parse_bootstrap_env('APEX_REPO_REF=' + value) + + def test_parentheses_and_hashes_remain_quoted_literals(self): + for quote in ('"', "'"): + for value in ('(main)', '#comment', 'main #comment'): + with self.subTest(quote=quote, value=value): + values = parse_bootstrap_env('APEX_REPO_REF=' + quote + value + quote) + self.assertEqual(values['APEX_REPO_REF'], value) + + +class TestUserDataContract(unittest.TestCase): + def test_public_repository_and_directory_defaults(self): + spec = normalize_user_data(example()) + self.assertEqual(spec.hostname, 'metis.a1.example.com') + self.assertEqual(spec.repository.name, 'metis.example.com') + self.assertIsNone(spec.repository.ref) + self.assertEqual([tier.mode for tier in spec.tiers], ['directory'] * 3) + self.assertEqual(spec.tiers[0].number, 1) + + def test_ref_and_explicit_storage_intent(self): + doc = add_env(example(), + 'APEX_REPO_REF="refs/tags/v1"\n' + 'APEX_TIER1_MODE=create\nAPEX_TIER1_DEVICE=/dev/disk/by-id/disk-a\n' + 'APEX_TIER2_MODE=import\nAPEX_TIER2_DEVICE=/dev/disk/by-id/disk-b\n' + 'APEX_TIER2_POOL_GUID=1234567\n') + spec = normalize_user_data(doc) + self.assertEqual(spec.repository.ref, 'refs/tags/v1') + self.assertEqual(spec.tiers[0].mode, 'create') + self.assertEqual(spec.tiers[1].pool_guid, '1234567') + + def test_legacy_device_requires_explicit_mode(self): + with self.assertRaisesRegex(ContractError, 'explicit mode'): + normalize_user_data(add_env(example(), 'APEX_TIER1_DEVICE=/dev/vdb\n')) + + def test_storage_conflicts_are_rejected(self): + cases = [ + 'APEX_TIER1_MODE=create\n', + 'APEX_TIER1_MODE=import\nAPEX_TIER1_DEVICE=/dev/vdb\n', + 'APEX_TIER1_MODE=directory\nAPEX_TIER1_DEVICE=/dev/vdb\n', + 'APEX_TIER1_MODE=create\nAPEX_TIER1_DEVICE=/dev/vdb\nAPEX_TIER1_POOL_GUID=1\n', + 'APEX_TIER1_MODE=erase\n', + 'APEX_TIER1_MODE=create\nAPEX_TIER1_DEVICE=/dev/vdb\n' + 'APEX_TIER2_MODE=create\nAPEX_TIER2_DEVICE=/dev/vdb\n', + ] + for assignment in cases: + with self.subTest(assignment=assignment): + with self.assertRaises(ContractError): + normalize_user_data(add_env(example(), assignment)) + + def test_arbitrary_cloud_init_operations_are_rejected(self): + for field in ('packages', 'bootcmd', 'disk_setup'): + doc = example() + doc[field] = [] + with self.subTest(field=field): + with self.assertRaises(ContractError): + normalize_user_data(doc) + doc = example() + doc['runcmd'] = ['touch /tmp/unapproved'] + with self.assertRaises(ContractError): + normalize_user_data(doc) + + def test_standard_list_form_of_bootstrap_command(self): + doc = example() + doc['runcmd'] = [['/usr/local/bin/apex-bootstrap']] + self.assertEqual(normalize_user_data(doc).hostname, doc['hostname']) + + def test_invalid_administrator_and_missing_keys(self): + changes = [ + ('name', 'root'), ('shell', '/bin/sh'), ('groups', ['sudo']), + ('sudo', 'ALL=(ALL) ALL'), ('ssh_authorized_keys', []), + ] + for key, value in changes: + doc = example() + doc['users'][0][key] = value + with self.subTest(key=key): + with self.assertRaises(ContractError): + normalize_user_data(doc) + + def test_unapproved_or_duplicate_file_writes(self): + for mutate in ('path', 'owner', 'permissions', 'append', 'duplicate'): + doc = example() + item = doc['write_files'][0] + if mutate == 'path': + item['path'] = '/etc/shadow' + elif mutate == 'owner': + item['owner'] = 'adam:adam' + elif mutate == 'permissions': + item['permissions'] = '0644' + elif mutate == 'append': + item['append'] = True + else: + doc['write_files'].append(copy.deepcopy(item)) + with self.subTest(mutate=mutate): + with self.assertRaises(ContractError): + normalize_user_data(doc) + + def test_rejects_wrong_types_and_missing_required_fields(self): + cases = [None, [], {'hostname': 'x.example.com'}] + for value in cases: + with self.subTest(value=value): + with self.assertRaises(ContractError): + normalize_user_data(value) + doc = example() + doc['preserve_hostname'] = 'false' + with self.assertRaises(ContractError): + normalize_user_data(doc) + + def test_repository_paths_and_embedded_passwords(self): + for name, url in [ + ('../escape', 'https://example.com/repo.git'), + ('metis', 'https://user:SECRET_SENTINEL@example.com/repo.git'), + ('metis', 'file:///etc/shadow'), + ('metis', '-uSECRET_SENTINEL'), + ]: + doc = example() + doc['write_files'][0]['content'] = ( + f'APEX_REPO_NAME="{name}"\nAPEX_REPO_URL="{url}"\n') + with self.subTest(name=name, url=url): + with self.assertRaises(ContractError) as caught: + normalize_user_data(doc) + self.assertNotIn('SECRET_SENTINEL', str(caught.exception)) + + def test_runtime_secrets_are_excluded_from_repr_and_identity(self): + doc = example() + for path, content, permission in [ + ('/home/adam/.ssh/id_ed25519', 'SECRET_SENTINEL', '0600'), + ('/home/adam/.ssh/known_hosts', 'HOST_TRUST_SENTINEL', '0644'), + ]: + doc['write_files'].append({ + 'path': path, 'owner': 'adam:adam', 'permissions': permission, + 'defer': True, 'content': content, + }) + spec = normalize_user_data(doc) + text = repr(spec) + repr(spec.identity()) + self.assertNotIn('SECRET_SENTINEL', text) + self.assertNotIn('PUBLIC_TEST_KEY', text) + self.assertNotIn('HOST_TRUST_SENTINEL', text) + self.assertEqual(spec.files[1].content, 'SECRET_SENTINEL') + + def test_administrator_files_require_defer(self): + doc = example() + doc['write_files'].append({ + 'path': '/home/adam/.ssh/known_hosts', 'owner': 'adam:adam', + 'permissions': '0644', 'content': 'example.com ssh-ed25519 VALUE', + }) + with self.assertRaises(ContractError): + normalize_user_data(doc) + + def test_contract_does_not_mutate_caller_input(self): + doc = example() + before = copy.deepcopy(doc) + normalize_user_data(doc) + self.assertEqual(doc, before) + + def test_identity_changes_for_target_but_not_credentials(self): + doc = example() + before = normalize_user_data(doc).identity() + doc['users'][0]['ssh_authorized_keys'] = ['ssh-ed25519 ANOTHER_PUBLIC_KEY operator'] + self.assertEqual(normalize_user_data(doc).identity(), before) + doc['hostname'] = 'other.a1.example.com' + self.assertNotEqual(normalize_user_data(doc).identity(), before) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_provision_image.py b/tests/test_provision_image.py new file mode 100644 index 0000000..cf601fe --- /dev/null +++ b/tests/test_provision_image.py @@ -0,0 +1,37 @@ +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from engine.provisioning.image import cloud_enroll, sanitize_files +from engine.provisioning.state import ProvisionError + + +class ImageTests(unittest.TestCase): + def test_incompatible_image_stops_before_any_enrollment(self): + for metadata in ('{}', '{"enrollment_interface":999}'): + with self.subTest(metadata=metadata), patch.object(Path, 'read_text', return_value=metadata), \ + patch('engine.provisioning.target.enroll') as enroll: + with self.assertRaisesRegex(ProvisionError, 'image enrollment interface'): + cloud_enroll() + enroll.assert_not_called() + + def test_sanitization_removes_cloned_identity_and_enrollment_but_preserves_runner(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary).resolve() + for relative in ('etc/ssh/ssh_host_ed25519_key', 'etc/ssh/ssh_host_ed25519_key.pub', + 'root/.ssh/authorized_keys', 'home/debian/.ssh/authorized_keys', + 'etc/machine-id', 'var/lib/dbus/machine-id', + 'var/lib/apex-init/enrollment.json', 'var/lib/cloud/instance/user-data.txt', + 'usr/local/lib/apex-provisioner/digest/engine/provisioning/target.py'): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('build-specific') + sanitize_files(root) + self.assertEqual(list((root / 'etc/ssh').glob('ssh_host_*')), []) + self.assertEqual((root / 'etc/machine-id').read_text(), '') + self.assertFalse((root / 'root/.ssh/authorized_keys').exists()) + self.assertFalse((root / 'var/lib/apex-init').exists()) + self.assertFalse((root / 'var/lib/cloud').exists()) + self.assertTrue((root / 'usr/local/lib/apex-provisioner/digest/engine/provisioning/target.py').is_file()) + self.assertTrue((root / 'etc/systemd/system/ssh.service.d/10-generate-host-keys.conf').is_file()) diff --git a/tests/test_provision_repository.py b/tests/test_provision_repository.py new file mode 100644 index 0000000..302ad3a --- /dev/null +++ b/tests/test_provision_repository.py @@ -0,0 +1,132 @@ +import os +from pathlib import Path +from types import SimpleNamespace +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from engine.provisioning.contract import Enrollment, Repository +from engine.provisioning.repository import reconcile_repository +from engine.provisioning.state import ProvisionError + + +class RepositoryTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.upstream = self.root / 'upstream' + self.commons = self.root / 'commons' + self.srv = self.root / 'srv' + self.srv.mkdir() + self.home = self.root / 'adam' + self.home.mkdir() + for path in (self.upstream, self.commons): + path.mkdir() + self.git(['init', '-q'], path) + self.git(['config', 'user.name', 'Test'], path) + self.git(['config', 'user.email', 'test@example.invalid'], path) + (self.commons / 'marker').write_text('commons') + (self.commons / 'etc').mkdir() + (self.commons / 'etc/.vimrc').write_text('set number\n') + self.git(['add', '.'], self.commons) + self.git(['commit', '-qm', 'initial'], self.commons) + (self.upstream / 'node.env').write_text('APEX_NODE_FQDN=metis.example.com\nAPEX_ENROLLMENT_INTERFACE=1\n') + (self.upstream / 'init.sh').write_text( + '#!/bin/sh\nset -eu\ngit config core.hooksPath commons/githooks\n' + 'printf \'export PATH="%s/commons:$PATH"\\n\' "$PWD" > "$HOME/.bashrc"\n' + 'ln -sf "$PWD/commons/etc/.vimrc" "$HOME/.vimrc"\n') + self.git(['-c', 'protocol.file.allow=always', 'submodule', 'add', str(self.commons), 'commons'], self.upstream) + self.git(['add', '.'], self.upstream) + self.git(['commit', '-qm', 'initial'], self.upstream) + self.spec = Enrollment('metis.example.com', Repository('metis', str(self.upstream), None), (), (), ()) + self.state = {} + self.saved = [] + self.account = patch('engine.provisioning.repository.pwd.getpwnam', + return_value=SimpleNamespace(pw_uid=os.getuid(), pw_gid=os.getgid(), pw_dir=str(self.home))) + self.account.start() + self.addCleanup(self.account.stop) + self.runner = patch('engine.provisioning.repository.as_adam', side_effect=self.local_run) + self.runner.start() + self.addCleanup(self.runner.stop) + + @staticmethod + def git(args, path): + return subprocess.run(['git', *args], cwd=path, text=True, capture_output=True, check=True).stdout.strip() + + def local_run(self, args, cwd=None): + if args[0] == 'git': + args = ['git', '-c', 'protocol.file.allow=always', *args[1:]] + return subprocess.run(args, cwd=cwd, text=True, capture_output=True, check=True, + env={**os.environ, 'HOME': str(self.home)}).stdout.strip() + + def enroll(self): + return reconcile_repository(self.spec, self.state, lambda: self.saved.append(True), self.srv) + + def test_repeat_does_not_follow_new_upstream_commit(self): + first = self.enroll() + (self.upstream / 'new').write_text('upstream advanced') + self.git(['add', '.'], self.upstream) + self.git(['commit', '-qm', 'advance'], self.upstream) + second = self.enroll() + self.assertEqual(first['commit'], second['commit']) + self.assertFalse((self.srv / 'metis' / 'new').exists()) + self.assertTrue(self.state['repository']['hook_complete']) + + def test_local_changes_are_preserved_and_stop_initialization(self): + self.enroll() + path = self.srv / 'metis' / 'node.env' + path.write_text('local changes') + with self.assertRaisesRegex(ProvisionError, 'local changes'): + self.enroll() + self.assertEqual(path.read_text(), 'local changes') + + def test_unrecorded_checkout_is_not_adopted(self): + (self.srv / 'metis').mkdir() + with self.assertRaisesRegex(ProvisionError, 'adoption'): + self.enroll() + + def test_noop_hook_cannot_claim_success(self): + (self.upstream / 'init.sh').write_text('true\n') + self.git(['add', '.'], self.upstream) + self.git(['commit', '-qm', 'noop'], self.upstream) + with self.assertRaises((ProvisionError, subprocess.CalledProcessError)): + self.enroll() + self.assertFalse(self.state['repository'].get('hook_complete', False)) + + def test_missing_or_unsupported_interface_stops_before_hook(self): + for version in ('', 'APEX_ENROLLMENT_INTERFACE=999\n'): + with self.subTest(version=version): + (self.upstream / 'node.env').write_text('APEX_NODE_FQDN=metis.example.com\n' + version) + self.git(['add', '.'], self.upstream) + self.git(['commit', '-qm', 'unsupported interface'], self.upstream) + self.spec = Enrollment('metis.example.com', Repository('missing-' + str(len(version)), str(self.upstream), None), (), (), ()) + self.state = {} + with self.assertRaisesRegex(ProvisionError, 'enrollment interface'): + self.enroll() + self.assertFalse((self.home / '.bashrc').exists()) + self.assertNotIn('hook_complete', self.state['repository']) + + def test_retry_checks_hook_state_instead_of_completion_marker(self): + self.enroll() + self.git(['config', 'core.hooksPath', 'wrong-path'], self.srv / 'metis') + with self.assertRaisesRegex(ProvisionError, 'hook configuration'): + self.enroll() + + def test_ordinary_commons_directory_is_not_a_pinned_submodule(self): + self.git(['rm', '-f', 'commons'], self.upstream) + (self.upstream / 'commons').mkdir() + (self.upstream / 'commons/ordinary').write_text('no submodule') + self.git(['add', '.'], self.upstream) + self.git(['commit', '-qm', 'ordinary directory'], self.upstream) + with self.assertRaisesRegex(ProvisionError, 'pinned Git submodule'): + self.enroll() + + def test_mismatched_node_hostname_stops_before_hook(self): + (self.upstream / 'node.env').write_text('APEX_NODE_FQDN=wrong.example.com\n') + self.git(['add', '.'], self.upstream) + self.git(['commit', '-qm', 'wrong'], self.upstream) + with self.assertRaisesRegex(ProvisionError, 'hostname'): + self.enroll() + self.assertNotIn('hook_complete', self.state['repository']) diff --git a/tests/test_provision_security.py b/tests/test_provision_security.py new file mode 100644 index 0000000..2f0380b --- /dev/null +++ b/tests/test_provision_security.py @@ -0,0 +1,544 @@ +import base64 +import copy +import hashlib +import json +import ipaddress +import re +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +from engine.provisioning.access import Host +from engine.provisioning import security + + +# Pinned helper default output, kept independent of the adapter's validation code. +HELPER_V4 = """# BEGIN UFW AND DOCKER +*filter +:ufw-user-forward - [0:0] +:ufw-docker-logging-deny - [0:0] +:DOCKER-USER - [0:0] +-A DOCKER-USER -j ufw-user-forward +-A DOCKER-USER -m conntrack --ctstate RELATED,ESTABLISHED -j RETURN +-A DOCKER-USER -m conntrack --ctstate INVALID -j DROP +-A DOCKER-USER -i docker0 -o docker0 -j ACCEPT +-A DOCKER-USER -j RETURN -s 10.0.0.0/8 +-A DOCKER-USER -j RETURN -s 172.16.0.0/12 +-A DOCKER-USER -j RETURN -s 192.168.0.0/16 +-A DOCKER-USER -j ufw-docker-logging-deny -m conntrack --ctstate NEW -d 10.0.0.0/8 +-A DOCKER-USER -j ufw-docker-logging-deny -m conntrack --ctstate NEW -d 172.16.0.0/12 +-A DOCKER-USER -j ufw-docker-logging-deny -m conntrack --ctstate NEW -d 192.168.0.0/16 +-A DOCKER-USER -j RETURN +-A ufw-docker-logging-deny -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW DOCKER BLOCK] " +-A ufw-docker-logging-deny -j DROP +COMMIT +# END UFW AND DOCKER +""" +HELPER_V6 = """# BEGIN UFW AND DOCKER +*filter +:ufw6-user-forward - [0:0] +:ufw6-docker-logging-deny - [0:0] +:DOCKER-USER - [0:0] +-A DOCKER-USER -j ufw6-user-forward +-A DOCKER-USER -m conntrack --ctstate RELATED,ESTABLISHED -j RETURN +-A DOCKER-USER -m conntrack --ctstate INVALID -j DROP +-A DOCKER-USER -i docker0 -o docker0 -j ACCEPT +-A DOCKER-USER -j RETURN -s fd00::/8 +-A DOCKER-USER -j ufw6-docker-logging-deny -m conntrack --ctstate NEW -d fd00::/8 +-A DOCKER-USER -j RETURN +-A ufw6-docker-logging-deny -m limit --limit 3/min --limit-burst 10 -j LOG --log-prefix "[UFW DOCKER BLOCK] " +-A ufw6-docker-logging-deny -j DROP +COMMIT +# END UFW AND DOCKER +""" + + +def native_alert(value, scope='Ip', origin='cscli', identifier=1): + return {'id': identifier, 'created_at': '2026-10-10T12:00:00Z', 'events_count': 1, + 'events': [], 'message': 'manual decision', 'scenario': 'manual', 'simulated': False, + 'source': {'scope': scope, 'value': value}, + 'decisions': [{'id': identifier, 'duration': '4h', 'origin': origin, 'scenario': 'manual', + 'scope': scope, 'simulated': False, 'type': 'ban', 'value': value}]} + + +class SecurityHost(Host): + """Keep filesystem effects real; substitute Debian commands and services.""" + def __init__(self, root): + super().__init__(root) + self.active = False + self.services = {name: {'LoadState': 'loaded', 'ActiveState': 'inactive', + 'UnitFileState': 'disabled', 'DropInPaths': '', + 'FragmentPath': '/usr/lib/systemd/system/' + name + '.service'} + for name in ('ufw', 'crowdsec', 'crowdsec-firewall-bouncer')} + self.commands = [] + self.decisions = [] + self.runtime_overrides = {} + self.forward_hook = True + self.fail = None + self.after_command = None + contents = { + '/etc/default/ufw': 'IPV6=yes\nDEFAULT_INPUT_POLICY="ACCEPT"\nDEFAULT_OUTPUT_POLICY="ACCEPT"\nDEFAULT_FORWARD_POLICY="DROP"\n', + '/etc/ufw/ufw.conf': 'ENABLED=no\n', + '/etc/ufw/user.rules': '*filter\nCOMMIT\n', + '/etc/ufw/user6.rules': '*filter\nCOMMIT\n', + '/etc/ufw/before.rules': '*filter\nCOMMIT\n', + '/etc/ufw/before6.rules': '*filter\nCOMMIT\n', + '/etc/ufw/after.rules': '*filter\nCOMMIT\n', + '/etc/ufw/after6.rules': '*filter\nCOMMIT\n', + '/usr/local/bin/ufw-docker': '#!/bin/sh\n', + } + for name, content in contents.items(): + path = self.path(name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + self.conffiles = {'/etc/default/ufw': hashlib.md5(contents['/etc/default/ufw'].encode()).hexdigest()} + checksums = [] + for name, content in contents.items(): + if not name.startswith('/etc/ufw/'): + continue + filename = name.rsplit('/', 1)[1] + template = '/usr/share/ufw/' + ('' if filename == 'ufw.conf' else 'iptables/') + filename + path = self.path(template) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + checksums.append(hashlib.md5(content.encode()).hexdigest() + ' ' + template.lstrip('/')) + manifest = self.path('/var/lib/dpkg/info/ufw.md5sums') + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text('\n'.join(checksums) + '\n') + + def command(self, args, check=True): + self.commands.append(args) + output, code = '', 0 + if args[:2] == ['systemctl', 'show']: + output = '\n'.join(k + '=' + v for k, v in self.services[args[2]].items()) + elif args[:2] == ['ufw', 'status']: + output = 'Status: ' + ('active' if self.active else 'inactive') + elif args[0] == 'dpkg-query': + output = '\n'.join(' ' + name + ' ' + digest for name, digest in self.conffiles.items()) + elif args[0] in ('iptables', 'ip6tables'): + if args[1] == '-C': + code = 0 if self.forward_hook else 1 + elif self.active and '# BEGIN UFW AND DOCKER' in self.path('/etc/ufw/after.rules').read_text(): + fixture = HELPER_V6 if args[0] == 'ip6tables' else HELPER_V4 + chain = args[2] + output = self.runtime_overrides.get((args[0], chain), '-N ' + chain + '\n' + '\n'.join( + line for line in fixture.splitlines() if line.startswith('-A ' + chain + ' ')) + '\n') + else: + output = '-N DOCKER-USER\n-A DOCKER-USER -j RETURN\n' + elif args[0] == '/usr/local/bin/ufw-docker': + if args[1] == 'check': + code = 0 if '# BEGIN UFW AND DOCKER' in self.path('/etc/ufw/after.rules').read_text() else 1 + elif args[1] == 'install': + for name in ('after.rules', 'after6.rules'): + path = self.path('/etc/ufw/' + name) + path.write_text(path.read_text() + (HELPER_V6 if name == 'after6.rules' else HELPER_V4)) + elif args[0] == 'ufw': + if 'allow' in args and 'proto' in args: + source, port = args[args.index('from') + 1], args[args.index('port') + 1] + v6 = ipaddress.ip_network(source, strict=False).version == 6 + path = self.path('/etc/ufw/user6.rules' if v6 else '/etc/ufw/user.rules') + line = '### tuple ### allow tcp ' + port + (' ::/0' if v6 else ' 0.0.0.0/0') + ' any ' + source + ' in comment=' + args[-1].encode().hex() + '\n' + path.write_text(path.read_text().replace(line, '') if 'delete' in args else path.read_text() + line) + elif args[1] == 'default': + key = {'incoming': 'INPUT', 'outgoing': 'OUTPUT', 'routed': 'FORWARD'}[args[3]] + path = self.path('/etc/default/ufw') + path.write_text(re.sub('DEFAULT_' + key + '_POLICY="[A-Z]+"', 'DEFAULT_' + key + '_POLICY="' + ('DROP' if args[2] == 'deny' else 'ACCEPT') + '"', path.read_text())) + elif args[-1] in ('enable', 'disable'): + self.active = args[-1] == 'enable' + self.path('/etc/ufw/ufw.conf').write_text('ENABLED=' + ('yes' if self.active else 'no') + '\n') + elif args[-1] != 'reload': + raise AssertionError(args) + elif args[0] == 'systemctl': + field, value = {'enable': ('UnitFileState', 'enabled'), 'disable': ('UnitFileState', 'disabled'), + 'start': ('ActiveState', 'active'), 'stop': ('ActiveState', 'inactive')}[args[1]] + self.services[args[2]][field] = value + if args[1:3] == ['stop', 'ufw']: + self.active = False + elif args[0] == 'crowdsec': + pass + elif args[:2] == ['cscli', 'lapi']: + code = 0 if self.services['crowdsec']['ActiveState'] == 'active' else 1 + elif args[:2] == ['cscli', 'decisions']: + alerts = self.decisions + if isinstance(alerts, list) and all(isinstance(a, dict) and isinstance(a.get('decisions'), list) for a in alerts): + if '--all' not in args: + alerts = [a for a in alerts if all(d['origin'] != 'CAPI' for d in a['decisions'])] + limit = int(args[args.index('--limit') + 1]) if '--limit' in args else 100 + if limit: + alerts = alerts[:limit] + output = json.dumps(alerts) + else: + raise AssertionError('Unexpected command: ' + repr(args)) + result = subprocess.CompletedProcess(args, code, output, '') + if self.after_command: + self.after_command(args) + if check and code: + raise RuntimeError('simulated command failure') + return result + + +class SecurityTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.host = SecurityHost(Path(self.temp.name)) + self.adapter = security.Security(self.host) + self.addCleanup(patch.stopall) + patch.object(security, 'HELPER_SHA256', hashlib.sha256(b'#!/bin/sh\n').hexdigest(), create=True).start() + + def test_masked_security_service_refuses_before_mutation(self): + self.host.services['crowdsec']['UnitFileState'] = 'masked' + before = self.host.path('/etc/default/ufw').read_bytes() + with self.assertRaises(security.SecurityError): + self.adapter.preflight() + self.assertEqual(self.host.path('/etc/default/ufw').read_bytes(), before) + self.assertFalse(self.host.active) + + def test_inactive_custom_policy_is_not_mistaken_for_pristine(self): + path = self.host.path('/etc/default/ufw') + path.write_text(path.read_text() + 'MANAGED_BY_ADMIN=yes\n') + with self.assertRaises(security.SecurityError): + self.adapter.preflight() + self.assertFalse(self.host.active) + + def test_native_generated_ufw_files_are_verified_against_package_templates(self): + self.assertEqual(set(self.host.conffiles), {'/etc/default/ufw'}) + self.assertTrue(self.adapter.preflight()['pristine']) + + def test_modified_generated_file_or_package_template_is_not_pristine(self): + for name in ('/etc/ufw/user.rules', '/usr/share/ufw/iptables/after.rules'): + with self.subTest(path=name): + path = self.host.path(name) + before = path.read_bytes() + path.write_bytes(before + b'# modified\n') + with self.assertRaises(security.SecurityError): + self.adapter.preflight() + path.write_bytes(before) + + def test_active_custom_policy_and_ipv6_rules_are_preserved_in_plan(self): + self.host.active = True + self.host.path('/etc/default/ufw').write_text('IPV6=yes\nDEFAULT_FORWARD_POLICY="DROP"\n') + self.host.path('/etc/ufw/user6.rules').write_text('# existing administrator sentinel\n') + plan = self.adapter.preflight() + self.assertFalse(plan['pristine']) + self.assertTrue(plan['ipv6']) + self.assertIn('sentinel', base64.b64decode(plan['files']['/etc/ufw/user6.rules']['data']).decode()) + + def test_incomplete_helper_is_rejected(self): + self.host.active = True + self.host.path('/etc/ufw/after.rules').write_text('# BEGIN UFW AND DOCKER\ncustom\n# END UFW AND DOCKER\n') + with self.assertRaises(security.SecurityError): + self.adapter.preflight() + + def prepared(self): + self.saved = [] + state = self.adapter.snapshot(self.adapter.preflight(), 22, '192.0.2.10') + self.saved.append(copy.deepcopy(state)) + return state + + def persist(self, state): + self.saved.append(copy.deepcopy(state)) + + def test_pristine_activation_establishes_security_without_losing_bootstrap(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + self.assertTrue(self.host.active) + self.assertIn('DEFAULT_INPUT_POLICY="DROP"', self.host.path('/etc/default/ufw').read_text()) + self.assertIn('DEFAULT_FORWARD_POLICY="ACCEPT"', self.host.path('/etc/default/ufw').read_text()) + rules = self.host.path('/etc/ufw/user.rules').read_text() + self.assertIn('192.0.2.10', rules) + self.assertIn('2222', rules) + for service in self.host.services.values(): + self.assertEqual(service['ActiveState'], 'active') + self.assertEqual(service['UnitFileState'], 'enabled') + self.adapter.verify(state) + + def test_active_policy_survives_activation_and_rollback(self): + self.host.active = True + self.host.services['ufw'].update(ActiveState='active', UnitFileState='enabled') + self.host.path('/etc/ufw/user6.rules').write_text('# IPv6 administrator sentinel\n') + before = {name: self.host.path(name).read_bytes() for name in security.FILES} + state = self.prepared() + self.adapter.activate(state, self.persist) + self.assertEqual(self.host.path('/etc/default/ufw').read_bytes(), before['/etc/default/ufw']) + self.adapter.restore(state, self.persist) + self.assertTrue(self.host.active) + for name, content in before.items(): + self.assertEqual(self.host.path(name).read_bytes(), content) + + def test_peer_range_ban_refuses_bouncer_and_can_restore_access(self): + self.host.decisions = [native_alert('192.0.2.0/24', 'Range')] + state = self.prepared() + with self.assertRaises(security.SecurityError): + self.adapter.activate(state, self.persist) + self.assertEqual(self.host.services['crowdsec-firewall-bouncer']['ActiveState'], 'inactive') + self.adapter.restore(state, self.persist) + self.assertFalse(self.host.active) + self.assertEqual(self.host.services['crowdsec']['ActiveState'], 'inactive') + + def test_concurrent_edit_is_preserved_and_inactive_access_recovers(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + path = self.host.path('/etc/default/ufw') + path.write_text(path.read_text() + 'ADMIN_SENTINEL=yes\n') + with self.assertRaises(security.SecurityError): + self.adapter.restore(state, self.persist) + self.assertFalse(self.host.active) + self.assertIn('ADMIN_SENTINEL=yes', path.read_text()) + self.assertTrue(state['access_recovered']) + self.assertEqual(state['cleanup'], 'pending') + self.assertEqual(self.host.services['crowdsec-firewall-bouncer']['ActiveState'], 'inactive') + + def test_finalization_removes_only_temporary_bootstrap_allowance(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + self.adapter.finalize(state, self.persist) + rules = self.host.path('/etc/ufw/user.rules').read_text() + self.assertNotIn('192.0.2.10', rules) + self.assertIn('2222', rules) + self.adapter.verify(state) + + def test_clean_rollback_restores_inactive_service_states(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + self.adapter.restore(state, self.persist) + for service in self.host.services.values(): + self.assertEqual(service['ActiveState'], 'inactive') + self.assertEqual(service['UnitFileState'], 'disabled') + self.assertEqual(state['cleanup'], 'complete') + + def test_rollback_does_not_restart_preexisting_service_stopped_by_administrator(self): + self.host.services['crowdsec'].update(ActiveState='active', UnitFileState='enabled') + state = self.prepared() + self.adapter.activate(state, self.persist) + self.host.services['crowdsec']['ActiveState'] = 'inactive' + self.adapter.restore(state, self.persist) + self.assertEqual(self.host.services['crowdsec']['ActiveState'], 'inactive') + self.assertEqual(self.host.services['crowdsec']['UnitFileState'], 'enabled') + self.assertNotIn(['systemctl', 'start', 'crowdsec'], self.host.commands) + + def test_owned_enable_does_not_claim_preexisting_activity_field(self): + self.host.services['crowdsec']['ActiveState'] = 'active' + state = self.prepared() + self.adapter.activate(state, self.persist) + self.host.services['crowdsec']['ActiveState'] = 'inactive' + self.adapter.restore(state, self.persist) + self.assertEqual(self.host.services['crowdsec']['ActiveState'], 'inactive') + self.assertEqual(self.host.services['crowdsec']['UnitFileState'], 'disabled') + + def test_conflicting_service_state_is_preserved_and_reported_pending(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + self.host.services['crowdsec']['ActiveState'] = 'failed' + with self.assertRaises(security.SecurityError): + self.adapter.restore(state, self.persist) + self.assertEqual(self.host.services['crowdsec']['ActiveState'], 'failed') + self.assertTrue(state['access_recovered']) + self.assertIn('service:crowdsec:ActiveState', state['diagnostics']) + + def test_redirected_file_does_not_prevent_runtime_access_recovery(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + target = self.host.path('/etc/ufw/after.rules') + sentinel = self.host.path('/administrator-owned') + sentinel.write_text('do not touch\n') + target.unlink() + target.symlink_to(sentinel) + with self.assertRaises(security.SecurityError): + self.adapter.restore(state, self.persist) + self.assertFalse(self.host.active) + self.assertTrue(state['access_recovered']) + self.assertEqual(sentinel.read_text(), 'do not touch\n') + + def test_metadata_change_is_not_silently_overwritten(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + path = self.host.path('/etc/default/ufw') + path.chmod(0o600) + with self.assertRaises(security.SecurityError): + self.adapter.verify(state) + with self.assertRaises(security.SecurityError): + self.adapter.restore(state, self.persist) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertFalse(self.host.active) + + def test_process_death_after_native_write_preserves_unknown_bytes(self): + state = self.prepared() + def die(args): + if args[:2] == ['ufw', 'default']: + raise KeyboardInterrupt() + self.host.after_command = die + with self.assertRaises(KeyboardInterrupt): + self.adapter.activate(state, self.persist) + self.host.after_command = None + recovered = copy.deepcopy(self.saved[-1]) + with self.assertRaises(security.SecurityError): + security.Security(self.host).restore(recovered, self.persist) + self.assertFalse(self.host.active) + self.assertTrue(recovered['access_recovered']) + self.assertEqual(recovered['cleanup'], 'pending') + self.assertIn('interrupted-mutation', recovered['diagnostics']) + + def test_active_firewall_keeps_source_limited_bootstrap_on_ambiguous_recovery(self): + self.host.active = True + self.host.services['ufw'].update(ActiveState='active', UnitFileState='enabled') + state = self.prepared() + self.adapter.activate(state, self.persist) + self.adapter.finalize(state, self.persist) + path = self.host.path('/etc/ufw/user6.rules') + path.write_text(path.read_text() + '# concurrent IPv6 sentinel\n') + before_restore = len(self.host.commands) + with self.assertRaises(security.SecurityError): + self.adapter.restore(state, self.persist) + self.assertTrue(self.host.active) + self.assertTrue(state['access_recovered']) + self.assertIn('192.0.2.10', self.host.path('/etc/ufw/user.rules').read_text()) + self.assertIn('# concurrent IPv6 sentinel', path.read_text()) + self.assertNotIn(['ufw', 'disable'], self.host.commands) + self.assertNotIn(['ufw', 'reload'], self.host.commands[before_restore:]) + + def test_preexisting_ssh_rule_is_never_relabelled_or_deleted(self): + self.host.active = True + self.host.services['ufw'].update(ActiveState='active', UnitFileState='enabled') + path = self.host.path('/etc/ufw/user.rules') + sentinel = '### tuple ### allow tcp 2222 0.0.0.0/0 any 0.0.0.0/0 in comment=' + b'administrator'.hex() + '\n' + path.write_text(path.read_text() + sentinel) + state = self.prepared() + self.adapter.activate(state, self.persist) + self.adapter.finalize(state, self.persist) + self.assertIn(sentinel, path.read_text()) + self.adapter.restore(state, self.persist) + self.assertIn(sentinel, path.read_text()) + + def test_retry_verification_detects_stopped_bouncer(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + self.host.services['crowdsec-firewall-bouncer']['ActiveState'] = 'inactive' + with self.assertRaises(security.SecurityError): + self.adapter.verify(state) + + def test_redirected_ufw_enable_file_is_not_written_during_recovery(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + path = self.host.path('/etc/ufw/ufw.conf') + sentinel = self.host.path('/administrator-config') + sentinel.write_text('do not change\n') + path.unlink() + path.symlink_to(sentinel) + with self.assertRaises(security.SecurityError): + self.adapter.restore(state, self.persist) + self.assertEqual(sentinel.read_text(), 'do not change\n') + self.assertFalse(self.host.active) + self.assertTrue(state['access_recovered']) + + def test_compatible_existing_helper_is_preserved(self): + self.host.active = True + for name in ('after.rules', 'after6.rules'): + path = self.host.path('/etc/ufw/' + name) + path.write_text(path.read_text() + (HELPER_V6 if name == 'after6.rules' else HELPER_V4)) + state = self.prepared() + self.adapter.activate(state, self.persist) + self.assertNotIn([security.HELPER, 'install'], self.host.commands) + + def test_helper_check_success_does_not_accept_permissive_marker_block(self): + self.host.active = True + for name in ('after.rules', 'after6.rules'): + self.host.path('/etc/ufw/' + name).write_text( + '# BEGIN UFW AND DOCKER\n*filter\n-A DOCKER-USER -j ACCEPT\nCOMMIT\n# END UFW AND DOCKER\n') + self.assertEqual(self.host.command([security.HELPER, 'check']).returncode, 0) + with self.assertRaises(security.SecurityError): + self.adapter.preflight() + + def test_valid_helper_files_require_live_rules_and_forward_hook(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + for executable, chain, contents in ( + ('iptables', 'DOCKER-USER', '-N DOCKER-USER\n-A DOCKER-USER -j RETURN\n'), + ('ip6tables', 'DOCKER-USER', '-N DOCKER-USER\n-A DOCKER-USER -j ACCEPT\n'), + ('iptables', 'DOCKER-USER', '-N DOCKER-USER\n-A DOCKER-USER -j ACCEPT\n' + '\n'.join( + line for line in HELPER_V4.splitlines() if line.startswith('-A DOCKER-USER ')) + '\n'), + ('iptables', 'ufw-docker-logging-deny', '-N ufw-docker-logging-deny\n')): + with self.subTest(executable=executable, chain=chain): + self.host.runtime_overrides[(executable, chain)] = contents + with self.assertRaises(security.SecurityError): + self.adapter.verify(state) + self.host.runtime_overrides.clear() + self.host.forward_hook = False + with self.assertRaises(security.SecurityError): + self.adapter.verify(state) + + def test_native_runtime_option_order_is_accepted_without_reordering_rules(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + output = self.host.command(['iptables', '-S', 'DOCKER-USER']).stdout + output = output.replace('RELATED,ESTABLISHED', 'ESTABLISHED,RELATED').replace( + '-j RETURN -s 10.0.0.0/8', '-s 10.0.0.0/8 -j RETURN') + self.host.runtime_overrides[('iptables', 'DOCKER-USER')] = output + self.adapter.verify(state) + lines = output.splitlines() + lines[1], lines[-1] = lines[-1], lines[1] + self.host.runtime_overrides[('iptables', 'DOCKER-USER')] = '\n'.join(lines) + '\n' + with self.assertRaises(security.SecurityError): + self.adapter.verify(state) + + def test_existing_live_ban_is_rejected_before_firewall_changes(self): + self.host.services['crowdsec'].update(ActiveState='active', UnitFileState='enabled') + self.host.decisions = [native_alert('192.0.2.10')] + with self.assertRaises(security.SecurityError): + self.prepared() + self.assertFalse(self.host.active) + self.assertNotIn('2222', self.host.path('/etc/ufw/user.rules').read_text()) + + def test_ipv6_peer_requires_ipv6_firewall_support(self): + path = self.host.path('/etc/default/ufw') + path.write_text(path.read_text().replace('IPV6=yes', 'IPV6=no')) + self.host.conffiles['/etc/default/ufw'] = hashlib.md5(path.read_bytes()).hexdigest() + with self.assertRaises(security.SecurityError): + self.adapter.snapshot(self.adapter.preflight(), 22, '2001:db8::1') + + def test_decision_check_covers_alerts_after_default_limit_and_central_api(self): + self.host.services['crowdsec'].update(ActiveState='active', UnitFileState='enabled') + harmless = [native_alert('198.51.100.' + str(i), identifier=i) for i in range(1, 101)] + for origin in ('cscli', 'CAPI'): + with self.subTest(origin=origin): + self.host.decisions = harmless + [native_alert('192.0.2.0/24', 'Range', origin, 101)] + with self.assertRaises(security.SecurityError): + self.prepared() + self.assertFalse(self.host.active) + + def test_non_native_decision_json_is_rejected_instead_of_skipped(self): + self.host.services['crowdsec'].update(ActiveState='active', UnitFileState='enabled') + for malformed in ([{'scope': 'Ip', 'value': '192.0.2.10'}], [{}], {'decisions': []}, None): + with self.subTest(value=malformed): + self.host.decisions = malformed + with self.assertRaises(security.SecurityError): + self.prepared() + + def test_optional_simulated_field_is_not_required_for_native_decisions(self): + self.host.services['crowdsec'].update(ActiveState='active', UnitFileState='enabled') + alert = native_alert('198.51.100.3') + del alert['decisions'][0]['simulated'] + self.host.decisions = [alert] + self.prepared() + alert['decisions'][0]['value'] = '192.0.2.0/24' + alert['decisions'][0]['scope'] = 'Range' + with self.assertRaises(security.SecurityError): + self.prepared() + + def test_surgical_helper_cleanup_preserves_concurrent_metadata_and_text(self): + state = self.prepared() + self.adapter.activate(state, self.persist) + path = self.host.path('/etc/ufw/after.rules') + path.write_text(path.read_text() + '# administrator sentinel\n') + path.chmod(0o600) + with self.assertRaises(security.SecurityError): + self.adapter.restore(state, self.persist) + self.assertIn('# administrator sentinel', path.read_text()) + self.assertNotIn('# BEGIN UFW AND DOCKER', path.read_text()) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_provision_security_access.py b/tests/test_provision_security_access.py new file mode 100644 index 0000000..98b5b96 --- /dev/null +++ b/tests/test_provision_security_access.py @@ -0,0 +1,105 @@ +import tempfile +import unittest +from pathlib import Path + +from engine.provisioning.access import AccessError, Transition +from engine.provisioning.security import SecurityError +from test_provision_access import FakeHost, TOKEN + + +from provision_fakes import SecurityBoundary + + +class SecurityAccessTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.host = FakeHost(Path(self.temp.name)) + self.security = SecurityBoundary() + self.clock = 1000 + self.transition = Transition(self.host, now=lambda: self.clock) + self.transition.security = self.security + + def begin(self, port=22): + return self.transition.execute('begin', TOKEN, port, '192.0.2.10') + + def test_activation_failure_restores_original_access_and_security(self): + self.security.fail_activation = True + with self.assertRaises((AccessError, SecurityError)): + self.begin() + self.assertEqual(self.host.ports, [22]) + self.assertFalse(self.security.active) + self.assertFalse(self.host.armed) + self.assertEqual(self.transition.load()['status'], 'rolled_back') + + def test_security_cleanup_failure_does_not_prevent_ssh_recovery(self): + self.begin() + self.transition.execute('finalize', TOKEN) + self.security.fail_restore = True + self.clock += 301 + with self.assertRaises(AccessError): + self.transition.execute('tick') + self.assertEqual(self.host.ports, [22]) + self.assertTrue(self.host.armed) + self.assertEqual(self.transition.load()['status'], 'recovery_pending') + self.security.fail_restore = False + self.transition.execute('tick') + self.assertEqual(self.transition.load()['status'], 'rolled_back') + self.assertFalse(self.host.armed) + + def test_committed_retry_cannot_accept_stopped_security(self): + self.begin() + self.transition.execute('finalize', TOKEN) + self.transition.execute('commit', TOKEN) + self.security.active = False + with self.assertRaises((AccessError, SecurityError)): + self.begin(2222) + self.assertEqual(self.host.ports, [2222]) + + def test_existing_final_ssh_still_activates_security_with_watchdog(self): + self.host.existing_final = True + self.host.ports = [2222] + self.assertEqual(self.begin(2222)['status'], 'transition') + self.assertTrue(self.security.active) + self.assertTrue(self.host.armed) + self.assertEqual(self.host.ports, [2222]) + self.transition.execute('finalize', TOKEN) + self.transition.execute('commit', TOKEN) + self.assertFalse(self.host.path('/etc/systemd/system/ssh.service.d/apex-init.conf').exists()) + self.assertFalse(self.host.armed) + + def test_legacy_committed_access_gains_security_without_reopening_root(self): + self.begin() + self.transition.execute('finalize', TOKEN) + self.transition.execute('commit', TOKEN) + state = self.transition.load() + state.pop('security') + self.transition.save(state) + self.security.active = False + self.assertEqual(self.begin(2222)['status'], 'transition') + self.assertEqual(self.host.ports, [2222]) + self.assertTrue(self.security.active) + self.assertTrue(self.host.armed) + + def test_failed_ssh_restore_still_restores_security_and_retries(self): + self.begin() + restore = self.host.restore + self.host.restore = lambda state: (_ for _ in ()).throw(AccessError('restart failed')) + self.clock += 301 + with self.assertRaises(AccessError): + self.transition.execute('tick') + self.assertFalse(self.security.active) + self.assertTrue(self.host.armed) + self.host.restore = restore + self.transition.execute('tick') + self.assertEqual(self.host.ports, [22]) + self.assertEqual(self.transition.load()['status'], 'rolled_back') + + def test_security_failure_at_commit_keeps_rollback_armed(self): + self.begin() + self.transition.execute('finalize', TOKEN) + self.security.active = False + with self.assertRaises(SecurityError): + self.transition.execute('commit', TOKEN) + self.assertTrue(self.host.armed) + self.assertEqual(self.transition.load()['status'], 'final') diff --git a/tests/test_provision_state.py b/tests/test_provision_state.py new file mode 100644 index 0000000..bfdc80d --- /dev/null +++ b/tests/test_provision_state.py @@ -0,0 +1,80 @@ +import json +import os +from pathlib import Path +import tempfile +import unittest + +from engine.provisioning.state import (Journal, ProvisionError, atomic_write, + check_heartbeat, exclusive_lock) + + +class StateTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + + def test_atomic_private_file_and_symlink_rejection(self): + path = self.root / 'state' / 'record' + atomic_write(path, 'first') + atomic_write(path, 'second') + self.assertEqual(path.read_text(), 'second') + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + link = self.root / 'redirect' + link.symlink_to(path.parent) + with self.assertRaises(ProvisionError): + atomic_write(link / 'record', 'corruption') + self.assertEqual(path.read_text(), 'second') + + def test_lock_rejects_competing_initializer_and_releases(self): + path = self.root / 'lock' + with exclusive_lock(path): + with self.assertRaises(ProvisionError): + with exclusive_lock(path): + self.fail('competing lock acquired') + with exclusive_lock(path): + pass + + def test_failed_stage_is_retryable_and_success_is_reverified(self): + path = self.root / 'journal' + record = Journal(path, {'hostname': 'metis'}, 'rev', 'digest') + with self.assertRaises(RuntimeError): + record.stage('package', lambda: (_ for _ in ()).throw(RuntimeError('SECRET'))) + self.assertNotIn('SECRET', path.read_text()) + self.assertEqual(json.loads(path.read_text())['failed_stage'], 'package') + record = Journal(path, {'hostname': 'metis'}, 'rev', 'digest') + results = iter(({'version': 'one'}, {'version': 'two'})) + record.stage('package', lambda: next(results)) + record.stage('package', lambda: next(results)) + self.assertEqual(record.data['stages']['package']['result']['version'], 'two') + + def test_identity_or_bundle_change_cannot_retarget_partial_run(self): + path = self.root / 'journal' + Journal(path, {'hostname': 'metis'}, 'rev', 'digest').save() + for identity, digest in (({'hostname': 'other'}, 'digest'), ({'hostname': 'metis'}, 'other')): + with self.assertRaises(ProvisionError): + Journal(path, identity, 'rev', digest) + + def test_unsupported_interface_cannot_resume_and_revisions_survive_retry(self): + path = self.root / 'journal' + Journal(path, {'hostname': 'metis'}, 'source-rev', 'digest').save() + resumed = Journal(path, {'hostname': 'metis'}, 'source-rev', 'digest') + self.assertEqual(resumed.data['baseline_revision'], 'source-rev') + for version in (None, 999): + resumed.data['enrollment_interface'] = version + resumed.save() + with self.assertRaisesRegex(ProvisionError, 'enrollment interface'): + Journal(path, {'hostname': 'metis'}, 'source-rev', 'digest') + + def test_expired_heartbeat_prevents_next_operation(self): + beat = self.root / 'heartbeat' + beat.touch() + os.utime(beat, (100, 100)) + check_heartbeat(beat, now=110) + with self.assertRaises(ProvisionError): + check_heartbeat(beat, now=200) + record = Journal(self.root / 'journal', {}, 'rev', 'digest') + called = [] + with self.assertRaises(ProvisionError): + record.stage('next', lambda: called.append(True), beat) + self.assertEqual(called, []) diff --git a/tests/test_provision_storage.py b/tests/test_provision_storage.py new file mode 100644 index 0000000..1e0365b --- /dev/null +++ b/tests/test_provision_storage.py @@ -0,0 +1,453 @@ +import copy +import json +import os +from pathlib import Path +import subprocess +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from engine.provisioning.contract import Enrollment, Repository, Tier +from engine.provisioning import storage +from engine.provisioning.storage import StorageError, _persist_swap, reconcile_storage + + +def enrollment(*tiers): + return Enrollment('node.example.com', Repository('node', 'https://example.com/node', None), + tuple(tiers) or (Tier(1, 'directory', None, None),), (), ()) + + +class TestStorage(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.srv = self.root / 'srv' + self.srv.mkdir() + self.swap = self.root / 'swapfile' + self.fstab = self.root / 'fstab' + self.fstab.write_text('# existing entries\n') + self.meminfo = self.root / 'meminfo' + self.meminfo.write_text('MemTotal: 8388608 kB\n') + self.swaps = self.root / 'swaps' + self.calls = [] + self.state = {} + self.saved = [] + self.mounts = [{'target': '/', 'source': '/dev/root', 'fstype': 'ext4', 'maj:min': '8:0'}] + self.devices = [self.device('/dev/root', '8:0', 'ROOT', ['/']), + self.device('/dev/spare', '8:16', 'SPARE', [])] + self.signatures = [] + self.pool = None + self.pool_member = '/dev/spare' + self.pool_mountpoint = self.srv / 'tier-1.node' + self.pool_owner = None + self.import_listing = '' + self.import_full_path_listing = None + self.set_active_swaps(['/dev/existing-swap']) + self.activate_swap = True + self.create_hook = None + for name, value in [('SRV', self.srv), ('SWAPFILE', self.swap), + ('FSTAB', self.fstab), ('MEMINFO', self.meminfo), ('SWAPS', self.swaps)]: + patcher = patch('engine.provisioning.storage.' + name, value, create=True) + patcher.start() + self.addCleanup(patcher.stop) + patcher = patch('engine.provisioning.storage.subprocess.run', side_effect=self.run_command) + patcher.start() + self.addCleanup(patcher.stop) + patcher = patch('engine.provisioning.storage._device_number', side_effect=self.device_number) + patcher.start() + self.addCleanup(patcher.stop) + + @staticmethod + def device(name, number, serial, mounts): + return {'name': name, 'maj:min': number, 'type': 'disk', 'pkname': None, + 'mountpoints': mounts, 'fstype': None, 'wwn': None, + 'serial': serial, 'size': 1024 ** 4} + + def device_number(self, path): + aliases = {'/dev/alias': '/dev/spare'} + path = aliases.get(str(path), str(path)) + for item in self.devices: + if item['name'] == path: + return item['maj:min'] + raise StorageError('selected device is missing or is not a block device') + + def save(self): + self.saved.append(copy.deepcopy(self.state)) + + def set_active_swaps(self, paths): + rows = ['Filename\tType\tSize\tUsed\tPriority'] + for path in paths: + escaped = path.replace('\\', r'\134').replace(' ', r'\040').replace('\t', r'\011').replace('\n', r'\012') + rows.append(f'{escaped}\tfile\t2097148\t0\t-2') + self.swaps.write_text('\n'.join(rows) + '\n') + + def run_command(self, argv, **kwargs): + self.calls.append(argv) + output = '' + if argv[0] == 'findmnt': + output = json.dumps({'filesystems': self.mounts}) + elif argv[0] == 'lsblk': + output = json.dumps({'blockdevices': self.devices}) + elif argv[0] == 'wipefs': + output = json.dumps({'signatures': self.signatures}) + elif argv[:2] == ['zpool', 'list']: + if self.pool: + output = f"tier-1\t{self.pool['guid']}\tONLINE\n" + elif argv[:2] == ['zpool', 'status']: + output = (' pool: tier-1\n state: ONLINE\nconfig:\n\n' + '\tNAME STATE READ WRITE CKSUM\n' + '\ttier-1 ONLINE 0 0 0\n' + f'\t {self.pool_member} ONLINE 0 0 0\n\nerrors: No known data errors\n') + elif argv[:2] == ['zpool', 'create']: + self.assertTrue(self.saved) + self.assertEqual(self.saved[-1]['storage']['tiers']['1']['status'], 'intent') + if self.create_hook: + self.create_hook() + self.pool = {'guid': '12345'} + elif argv[:2] == ['zpool', 'import']: + if '-N' in argv: + self.pool = {'guid': argv[-1]} + else: + output = self.import_listing + if self.import_full_path_listing is not None and kwargs['env'].get('ZPOOL_VDEV_NAME_PATH') == '1' \ + and kwargs['env'].get('ZPOOL_VDEV_NAME_FOLLOW_LINKS') == '1' \ + and 'ZPOOL_VDEV_NAME_GUID' not in kwargs['env']: + output = self.import_full_path_listing + elif argv[:2] == ['zfs', 'list']: + mounted = 'yes' if any(m['source'] == 'tier-1' for m in self.mounts) else 'no' + output = f'tier-1\t{self.pool_mountpoint}\tfilesystem\t{mounted}\n' + elif argv[:2] == ['zfs', 'get']: + output = (self.pool_owner or self.state['storage']['identity']) + '\n' + elif argv[:2] == ['zfs', 'set']: + self.pool_owner = argv[2].split('=', 1)[1] + elif argv[:2] == ['zfs', 'mount']: + self.mounts.append({'target': str(self.srv / 'tier-1.node'), 'source': 'tier-1', + 'fstype': 'zfs', 'maj:min': '0:77'}) + elif argv[0] == 'mkswap': + self.assertTrue(self.swap.exists()) + self.assertEqual(self.saved[-1]['storage']['swap']['status'], 'intent') + elif argv == ['swapon', str(self.swap)]: + if self.activate_swap: + self.set_active_swaps([str(self.swap)]) + else: + self.fail('Unexpected command: ' + repr(argv)) + return subprocess.CompletedProcess(argv, 0, output, '') + + def test_directory_preserves_contents_and_only_reports_new_directory(self): + path = self.srv / 'tier-1.node' + path.mkdir() + (path / 'keep').write_text('existing data') + result = reconcile_storage(enrollment(Tier(1, 'directory', None, None), + Tier(2, 'directory', None, None)), self.state, self.save) + self.assertEqual((path / 'keep').read_text(), 'existing data') + self.assertFalse(result['tiers'][0]['created']) + self.assertTrue(result['tiers'][1]['created']) + self.assertEqual(result['swap']['status'], 'skipped') + + def test_directory_symlink_and_unexpected_mount_are_rejected(self): + path = self.srv / 'tier-1.node' + path.symlink_to(self.root) + with self.assertRaises(StorageError): + reconcile_storage(enrollment(), self.state, self.save) + path.unlink() + path.mkdir() + self.mounts.append({'target': str(path), 'source': '/dev/other', 'fstype': 'ext4', 'maj:min': '8:32'}) + with self.assertRaises(StorageError): + reconcile_storage(enrollment(), self.state, self.save) + + def test_pending_directory_ownership_survives_later_tier_failure_until_ack(self): + spec = enrollment(Tier(1, 'directory', None, None), Tier(2, 'directory', None, None)) + first = self.srv / 'tier-1.node' + second = self.srv / 'tier-2.node' + second.symlink_to(self.root) + with self.assertRaises(StorageError): + reconcile_storage(spec, self.state, self.save) + self.assertTrue(first.is_dir()) + self.state = copy.deepcopy(self.saved[-1]) + second.unlink() + second.mkdir() + result = reconcile_storage(spec, self.state, self.save) + self.assertEqual(result['ownership_pending'], [str(first)]) + self.assertEqual(reconcile_storage(spec, self.state, self.save)['ownership_pending'], [str(first)]) + storage.acknowledge_storage_ownership(spec, self.state, [str(first)], self.save) + self.state = copy.deepcopy(self.saved[-1]) + self.assertEqual(reconcile_storage(spec, self.state, self.save)['ownership_pending'], []) + + def test_replaced_pending_directory_is_never_adopted_or_acknowledged(self): + spec = enrollment() + path = self.srv / 'tier-1.node' + reconcile_storage(spec, self.state, self.save) + path.rename(self.srv / 'original') + path.mkdir() + with self.assertRaisesRegex(StorageError, 'identity'): + reconcile_storage(spec, self.state, self.save) + with self.assertRaisesRegex(StorageError, 'identity'): + storage.acknowledge_storage_ownership(spec, self.state, [str(path)], self.save) + + def test_preexisting_directory_cannot_be_acknowledged(self): + path = self.srv / 'tier-1.node' + path.mkdir() + spec = enrollment() + result = reconcile_storage(spec, self.state, self.save) + self.assertEqual(result['ownership_pending'], []) + with self.assertRaises(StorageError): + storage.acknowledge_storage_ownership(spec, self.state, [str(path)], self.save) + + def test_ownership_acknowledgement_rejects_retargeted_enrollment(self): + spec = enrollment() + result = reconcile_storage(spec, self.state, self.save) + changed = Enrollment('other.example.com', spec.repository, spec.tiers, (), ()) + with self.assertRaisesRegex(StorageError, 'identity'): + storage.acknowledge_storage_ownership(changed, self.state, result['ownership_pending'], self.save) + self.assertEqual(reconcile_storage(spec, self.state, self.save)['ownership_pending'], + result['ownership_pending']) + + def test_missing_root_and_occupied_device_never_create_pool(self): + for device in ('/dev/missing', '/dev/root', '/dev/spare'): + with self.subTest(device=device): + self.signatures = [{'type': 'ext4'}] if device == '/dev/spare' else [] + with self.assertRaises(StorageError): + reconcile_storage(enrollment(Tier(1, 'create', device, None)), {}, lambda: None) + self.assertFalse(any(c[:2] == ['zpool', 'create'] for c in self.calls)) + + def test_aliases_selected_by_two_tiers_are_rejected_before_creation(self): + with self.assertRaises(StorageError): + reconcile_storage(enrollment(Tier(1, 'create', '/dev/spare', None), + Tier(2, 'create', '/dev/alias', None)), self.state, self.save) + self.assertFalse(any(c[:2] == ['zpool', 'create'] for c in self.calls)) + + def test_create_records_guid_and_retry_verifies_without_recreation(self): + spec = enrollment(Tier(1, 'create', '/dev/spare', None)) + reconcile_storage(spec, self.state, self.save) + self.assertEqual(self.state['storage']['tiers']['1']['guid'], '12345') + reconcile_storage(spec, self.state, self.save) + self.assertEqual(sum(c[:2] == ['zpool', 'create'] for c in self.calls), 1) + self.devices[1]['serial'] = 'REPLACED' + with self.assertRaises(StorageError): + reconcile_storage(spec, self.state, self.save) + + def test_existing_pool_name_without_journal_is_not_ownership(self): + self.pool = {'guid': '12345'} + with self.assertRaises(StorageError): + reconcile_storage(enrollment(Tier(1, 'create', '/dev/spare', None)), self.state, self.save) + + def test_import_requires_exact_guid_and_single_device(self): + self.import_listing = (' pool: tier-1\n id: 54321\n state: ONLINE\n' + ' config:\n\n\ttier-1 ONLINE\n\t /dev/spare ONLINE\n') + with self.assertRaises(StorageError): + reconcile_storage(enrollment(Tier(1, 'import', '/dev/spare', '12345')), self.state, self.save) + self.assertFalse(any(c[:2] == ['zpool', 'import'] and '-N' in c for c in self.calls)) + + def test_unknown_swapfile_is_preserved(self): + self.set_active_swaps([]) + self.swap.write_text('do not overwrite') + result = reconcile_storage(enrollment(), self.state, self.save) + self.assertEqual(self.swap.read_text(), 'do not overwrite') + self.assertEqual(result['swap']['status'], 'skipped') + self.assertIn('existing', result['swap']['reason']) + + def test_root_disk_descendant_is_protected(self): + child = self.device('/dev/root1', '8:1', 'ROOT', ['/boot']) + child.update(type='part', pkname='/dev/root', partuuid='root-partition') + self.devices.append(child) + for device in ('/dev/root', '/dev/root1'): + with self.subTest(device=device): + with self.assertRaisesRegex(StorageError, 'intersects'): + reconcile_storage(enrollment(Tier(1, 'create', device, None)), self.state, self.save) + self.state.clear() + self.assertFalse(any(c[:2] == ['zpool', 'create'] for c in self.calls)) + + def test_missing_hardware_identity_stops_before_creation(self): + self.devices[1]['serial'] = None + with self.assertRaisesRegex(StorageError, 'stable'): + reconcile_storage(enrollment(Tier(1, 'create', '/dev/spare', None)), self.state, self.save) + self.assertFalse(any(c[:2] == ['zpool', 'create'] for c in self.calls)) + + def test_identity_rechecked_after_intent_is_saved(self): + def changed_device(): + self.save() + self.devices[1]['serial'] = 'REPLACED' + with self.assertRaisesRegex(StorageError, 'changed before'): + reconcile_storage(enrollment(Tier(1, 'create', '/dev/spare', None)), self.state, changed_device) + self.assertFalse(any(c[:2] == ['zpool', 'create'] for c in self.calls)) + + def test_created_whole_disk_pool_can_use_a_new_partition(self): + def partition_created(): + child = self.device('/dev/spare1', '8:17', 'SPARE', []) + child.update(type='part', pkname='/dev/spare') + self.devices.append(child) + self.pool_member = '/dev/spare1' + self.create_hook = partition_created + result = reconcile_storage(enrollment(Tier(1, 'create', '/dev/spare', None)), self.state, self.save) + self.assertEqual(result['tiers'][0]['guid'], '12345') + + def importable(self, members='\t spare ONLINE\n'): + self.import_listing = (' pool: tier-1\n id: 12345\n state: ONLINE\n' + ' action: The pool can be imported using its name or numeric identifier.\n' + ' config:\n\n\ttier-1 ONLINE\n' + members) + + def test_imports_declared_guid_without_force_or_mountpoint_rewrite(self): + self.importable() + result = reconcile_storage(enrollment(Tier(1, 'import', '/dev/spare', '12345')), self.state, self.save) + self.assertEqual(result['tiers'][0]['guid'], '12345') + self.assertFalse(result['tiers'][0]['created']) + self.assertEqual(result['ownership_pending'], []) + with self.assertRaises(StorageError): + storage.acknowledge_storage_ownership( + enrollment(Tier(1, 'import', '/dev/spare', '12345')), self.state, + [str(self.pool_mountpoint)], self.save) + self.assertIn(['zpool', 'import', '-N', '-d', '/dev/spare', '12345'], self.calls) + self.assertFalse(any('-f' in command for command in self.calls)) + self.assertFalse(any('mountpoint=' in arg for command in self.calls for arg in command)) + + def test_multidevice_import_is_rejected(self): + self.importable('\t spare ONLINE\n\t other ONLINE\n') + with self.assertRaisesRegex(StorageError, 'exactly one'): + reconcile_storage(enrollment(Tier(1, 'import', '/dev/spare', '12345')), self.state, self.save) + self.assertFalse(any(c[:2] == ['zpool', 'import'] and '-N' in c for c in self.calls)) + + def partitioned_import_fixture(self): + for suffix, number in (('1', '8:17'), ('9', '8:25')): + child = self.device('/dev/spare' + suffix, number, None, []) + child.update(type='part', pkname='/dev/spare', partuuid='partition-' + suffix) + self.devices.append(child) + self.pool_member = '/dev/spare1' + self.importable() + self.import_full_path_listing = self.import_listing.replace('spare ONLINE', '/dev/spare1 ONLINE') + + def test_import_discovery_preserves_partition_path_instead_of_whole_disk_display(self): + self.partitioned_import_fixture() + with patch.dict(os.environ, {'ZPOOL_VDEV_NAME_GUID': '1'}): + result = reconcile_storage(enrollment(Tier(1, 'import', '/dev/spare1', '12345')), + self.state, self.save) + self.assertEqual(result['tiers'][0]['guid'], '12345') + self.assertIn(['zpool', 'import', '-N', '-d', '/dev/spare1', '12345'], self.calls) + + def test_import_discovery_never_accepts_a_sibling_partition(self): + self.partitioned_import_fixture() + with self.assertRaisesRegex(StorageError, 'membership differs'): + reconcile_storage(enrollment(Tier(1, 'import', '/dev/spare9', '12345')), self.state, self.save) + self.assertFalse(any(c[:2] == ['zpool', 'import'] and '-N' in c for c in self.calls)) + + def test_import_discovery_parent_only_output_is_not_partition_proof(self): + self.partitioned_import_fixture() + self.import_full_path_listing = self.import_listing.replace('spare ONLINE', '/dev/spare ONLINE') + with self.assertRaisesRegex(StorageError, 'membership differs'): + reconcile_storage(enrollment(Tier(1, 'import', '/dev/spare1', '12345')), self.state, self.save) + self.assertFalse(any(c[:2] == ['zpool', 'import'] and '-N' in c for c in self.calls)) + + def test_import_does_not_mount_or_reassign_unrelated_mountpoint(self): + self.importable() + self.pool_mountpoint = '/important-data' + with self.assertRaisesRegex(StorageError, 'mountpoint'): + reconcile_storage(enrollment(Tier(1, 'import', '/dev/spare', '12345')), self.state, self.save) + self.assertFalse(any(c[:2] in (['zfs', 'mount'], ['zfs', 'set']) for c in self.calls)) + + def test_swap_skip_reports_low_space_and_unsupported_filesystems(self): + self.set_active_swaps([]) + with patch('engine.provisioning.storage.os.statvfs', return_value=SimpleNamespace(f_bavail=1, f_frsize=4096)): + result = reconcile_storage(enrollment(), self.state, self.save) + self.assertIn('insufficient', result['swap']['reason']) + self.mounts[0]['fstype'] = 'btrfs' + result = reconcile_storage(enrollment(), self.state, self.save) + self.assertIn('supported', result['swap']['reason']) + self.assertFalse(self.swap.exists()) + + def test_swap_sizes_allocation_and_records_verified_result(self): + self.set_active_swaps([]) + with patch('engine.provisioning.storage.os.statvfs', return_value=SimpleNamespace(f_bavail=16, f_frsize=1024 ** 3)), \ + patch('engine.provisioning.storage.os.posix_fallocate', create=True) as allocate, \ + patch('engine.provisioning.storage._persist_swap') as persist: + result = reconcile_storage(enrollment(), self.state, self.save) + self.assertEqual(allocate.call_args.args[1:], (0, 4 * 1024 ** 3)) + self.assertEqual(result['swap'], {'status': 'complete', 'size': 4 * 1024 ** 3}) + persist.assert_called_once() + + def test_existing_active_swap_preserved_without_cli_inspection(self): + before = self.swaps.read_text() + result = reconcile_storage(enrollment(), self.state, self.save) + self.assertEqual(result['swap'], {'status': 'skipped', 'reason': 'existing active swap preserved'}) + self.assertEqual(self.swaps.read_text(), before) + self.assertFalse(any(command[0] in ('swapon', 'mkswap') for command in self.calls)) + self.assertFalse(self.swap.exists()) + + def test_proc_swaps_empty_and_escaped_paths(self): + self.set_active_swaps([]) + self.assertEqual(storage._active_swaps(), []) + self.swaps.write_text('Filename\tType\tSize\tUsed\tPriority\n' + '/swap\\040file\\011name\\012with\\134slash\tfile\t2097148\t0\t-2\n') + self.assertEqual(storage._active_swaps(), ['/swap file\tname\nwith\\slash']) + + def test_unobserved_swap_activation_is_not_persisted_or_reported_complete(self): + self.set_active_swaps([]) + self.activate_swap = False + with patch('engine.provisioning.storage.os.statvfs', return_value=SimpleNamespace(f_bavail=16, f_frsize=1024 ** 3)), \ + patch('engine.provisioning.storage.os.posix_fallocate', create=True), \ + patch('engine.provisioning.storage._persist_swap') as persist: + result = reconcile_storage(enrollment(), self.state, self.save) + self.assertEqual(result['swap']['status'], 'skipped') + self.assertIn('activation did not become observable', result['swap']['reason']) + persist.assert_not_called() + + def test_malformed_proc_swaps_stops_optional_allocation(self): + for text in ('', 'wrong header\n', 'Filename Type Size Used Priority\n/swapfile file broken 0 -2\n'): + with self.subTest(text=text): + self.swaps.write_text(text) + result = reconcile_storage(enrollment(), self.state, self.save) + self.assertEqual(result['swap']['status'], 'skipped') + self.assertIn('swap inventory', result['swap']['reason']) + self.assertFalse(self.swap.exists()) + + def test_small_memory_swap_is_two_gib(self): + self.set_active_swaps([]) + self.meminfo.write_text('MemTotal: 4194304 kB\n') + with patch('engine.provisioning.storage.os.statvfs', return_value=SimpleNamespace(f_bavail=16, f_frsize=1024 ** 3)), \ + patch('engine.provisioning.storage.os.posix_fallocate', create=True) as allocate, \ + patch('engine.provisioning.storage._persist_swap'): + result = reconcile_storage(enrollment(), self.state, self.save) + self.assertEqual(allocate.call_args.args[1:], (0, 2 * 1024 ** 3)) + self.assertEqual(result['swap']['size'], 2 * 1024 ** 3) + + def test_fstab_preserves_existing_content_and_does_not_duplicate_swap(self): + actual_lstat = Path.lstat + + def root_owned(path, *args, **kwargs): + info = actual_lstat(path, *args, **kwargs) + if path == self.fstab: + fields = list(info) + fields[4] = 0 + return os.stat_result(fields) + return info + + with patch('engine.provisioning.storage.Path.lstat', root_owned), \ + patch('engine.provisioning.storage.os.fchown'): + _persist_swap() + _persist_swap() + self.assertEqual(self.fstab.read_text(), '# existing entries\n' + + f'{self.swap} none swap sw 0 0\n') + + def test_pool_guid_change_on_retry_is_rejected(self): + spec = enrollment(Tier(1, 'create', '/dev/spare', None)) + reconcile_storage(spec, self.state, self.save) + self.pool['guid'] = '67890' + with self.assertRaisesRegex(StorageError, 'GUID'): + reconcile_storage(spec, self.state, self.save) + + def test_interrupted_create_recovers_only_matching_pool_ownership(self): + spec = enrollment(Tier(1, 'create', '/dev/spare', None)) + reconcile_storage(spec, self.state, self.save) + record = self.state['storage']['tiers']['1'] + record['status'] = 'intent' + del record['guid'] + reconcile_storage(spec, self.state, self.save) + self.assertEqual(record['guid'], '12345') + self.assertEqual(sum(c[:2] == ['zpool', 'create'] for c in self.calls), 1) + self.pool_owner = 'other-enrollment' + with self.assertRaisesRegex(StorageError, 'ownership'): + reconcile_storage(spec, self.state, self.save) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_provision_target.py b/tests/test_provision_target.py new file mode 100644 index 0000000..bce9900 --- /dev/null +++ b/tests/test_provision_target.py @@ -0,0 +1,87 @@ +import contextlib +import io +import json +import os +from pathlib import Path +from types import SimpleNamespace +import tempfile +import unittest +from unittest.mock import patch + +from engine.provisioning import target +from engine.provisioning.contract import normalize_user_data +from engine.provisioning.state import ProvisionError, atomic_write, safe_path +from tests.test_provision_contract import example + + +class TargetTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + + def test_invalid_public_key_rejected_before_host_enrollment(self): + with self.assertRaisesRegex(ProvisionError, 'cryptographic validation'): + target.validate_keys(normalize_user_data(example())) + + def test_administrator_preserves_existing_keys_and_uses_narrow_writes(self): + home = self.root / 'home/adam' + (home / '.ssh').mkdir(parents=True) + (home / '.ssh/authorized_keys').write_text('existing-key\n') + account = SimpleNamespace(pw_dir='/home/adam', pw_uid=os.getuid(), pw_gid=os.getgid(), pw_shell='/bin/bash') + commands = [] + + def command(args, **kwargs): + commands.append(args) + if args == ['hostname']: + return 'provider.example.com' + if args == ['id', '-nG', 'adam']: + return 'adam sudo docker' + return '' + + def mapped(path): + path = Path(path) + return safe_path(path if path.is_relative_to(self.root) else self.root / str(path).lstrip('/')) + + def write(path, content, mode=0o600, **kwargs): + # Keep real file writes; privileged ownership is the isolated boundary. + kwargs.pop('uid', None) + kwargs.pop('gid', None) + atomic_write(mapped(path), content, mode, **kwargs) + + with patch.object(target.pwd, 'getpwnam', return_value=account), \ + patch.object(target, 'safe_path', side_effect=mapped), \ + patch.object(target, 'atomic_write', side_effect=write), \ + patch.object(target, 'run', side_effect=command): + spec = normalize_user_data(example()) + target.administrator(spec) + target.administrator(spec) + content = (home / '.ssh/authorized_keys').read_text().splitlines() + self.assertEqual(content, ['existing-key', 'ssh-ed25519 PUBLIC_TEST_KEY operator']) + self.assertIn(['hostnamectl', 'set-hostname', spec.hostname], commands) + self.assertEqual((self.root / 'etc/sudoers.d/99-apex-adam').stat().st_mode & 0o777, 0o440) + self.assertFalse((self.root / 'etc/machine-id').exists()) + self.assertFalse((self.root / 'etc/network').exists()) + + def test_completion_requires_committed_access_and_preserves_revisions(self): + state = self.root / 'state' + (state / 'access').mkdir(parents=True) + record = {'status': 'enrolled', 'repository': {'commit': 'nodecommit', 'commons_commit': 'commonscommit'}} + atomic_write(state / 'enrollment.json', json.dumps(record)) + atomic_write(state / 'access/state.json', json.dumps({'status': 'final'})) + with patch.object(target, 'STATE', state), patch.object(target, 'preflight'), \ + contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(target.main(['complete']), 1) + self.assertEqual(json.loads((state / 'enrollment.json').read_text())['status'], 'enrolled') + atomic_write(state / 'access/state.json', json.dumps({'status': 'committed'})) + self.assertEqual(target.main(['complete']), 0) + actual = json.loads((state / 'enrollment.json').read_text()) + self.assertEqual(actual['external_access'], 'verified') + self.assertEqual(actual['repository'], record['repository']) + + def test_preflight_rejects_wrong_os_before_any_enrollment(self): + with patch.object(target.os, 'geteuid', return_value=0), \ + patch.object(target.platform, 'machine', return_value='x86_64'), \ + patch.object(Path, 'read_text', return_value='ID=ubuntu\nVERSION_ID="24.04"\n'): + with self.assertRaisesRegex(ProvisionError, 'Debian 13'): + target.preflight() diff --git a/tests/test_provision_transport.py b/tests/test_provision_transport.py new file mode 100644 index 0000000..7e61f7c --- /dev/null +++ b/tests/test_provision_transport.py @@ -0,0 +1,517 @@ +import io +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +from engine.provisioning.transport import ( + SSH, TransportError, HandoverUncertain, build_bundle, destination, local_file, protected_handover, + initialize, + REMOTE_PATH_GUARD, remote_atomic_input, +) + + +class TransportTests(unittest.TestCase): + def test_durable_runner_checks_access_after_seed_before_enrollment(self): + for compatible in (False, True): + with self.subTest(compatible=compatible), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'user-data' + source.write_bytes(b'opaque input') + stage = root / 'stage' + stage.mkdir() + installed = root / 'bundle' + seed = installed / 'engine/provisioning/seed.sh' + seed.parent.mkdir(parents=True) + seed.write_text('printf "seed\\n" >> "$APEX_TEST_EVENTS"\n') + binaries = root / 'bin' + binaries.mkdir() + python = binaries / 'python3' + python.write_text('#!' + sys.executable + '\n' + ''' +import os, sys +module, operation = sys.argv[2:4] +event = 'access-' + operation if module.endswith('.access') else operation +with open(os.environ['APEX_TEST_EVENTS'], 'a') as stream: + stream.write(event + '\\n') +if event == 'access-preflight' and os.environ['APEX_TEST_COMPATIBLE'] == 'False': + sys.exit(42) +if event == 'enroll': + sys.exit(43) +''') + python.chmod(0o755) + events = root / 'events' + environment = dict(os.environ, PATH=str(binaries) + os.pathsep + os.environ['PATH'], + APEX_TEST_EVENTS=str(events), APEX_TEST_COMPATIBLE=str(compatible)) + class Remote: + user, host, identity = 'root', 'example.com', None + resolved_port = 22 + def __init__(self, *args): + pass + def resolve(self): + pass + def run(self, script, data=None, **kwargs): + value = b'' + if 'ssh_host_' in script: + value = b'192.0.2.1 50000 192.0.2.2 22\nssh-ed25519 AAAA fixture\n' + elif 'mktemp -d /var/tmp/apex-init.' in script: + value = b'/var/tmp/apex-init.0123456789\n' + elif 'systemd-run --quiet' in script: + arguments = shlex.split(script) + wrapper = arguments[-1].replace('/usr/local/lib/apex-provisioner/' + 'a' * 64, str(installed)) + wrapper = wrapper.replace('/var/tmp/apex-init.0123456789', str(stage)) + wrapper = wrapper.replace('/etc/tmpfiles.d/', str(root) + '/') + subprocess.run(['/bin/sh', '-c', wrapper], env=environment, + capture_output=True, timeout=30) + elif script.startswith('if test -f /var/tmp'): + value = (stage / 'result').read_bytes() + return subprocess.CompletedProcess([], 0, value, b'') + with patch('engine.provisioning.transport.SSH', Remote), \ + patch('engine.provisioning.transport.build_bundle', return_value=(b'bundle', 'a' * 64, 'b' * 40)), \ + patch('engine.provisioning.transport.shutil.which', return_value='/usr/bin/ssh'): + with self.assertRaises(TransportError): + initialize(root, 'root@example.com', source, progress=lambda message: None) + expected = ['seed', 'validate', 'access-preflight'] + if compatible: + expected.append('enroll') + self.assertEqual(events.read_text().splitlines(), expected) + self.assertEqual((stage / 'result').read_text().strip(), '43' if compatible else '42') + + def test_rejects_options_shell_fragments_and_invalid_ports(self): + for value in ['root@-oProxyCommand=id', 'root@host;id', '-oProxyCommand=id@host', + 'bob;id@host', 'bob name@host', 'bob\n@host', 'bob/other@host', + 'bob$(id)@host', '@host', '9bob@host', 'a' * 33 + '@host', + 'root@host\nfoo', 'host', 'debian@host -oProxyCommand=id']: + with self.subTest(value=value), self.assertRaises(TransportError): + destination(value) + for port in ['0', '65536', '-1', '22;id', '2.5', '']: + with self.subTest(port=port), self.assertRaises(TransportError): + destination('root@example.com', port) + self.assertEqual(destination('adam@node', 2222), ('adam', 'node', 2222)) + + def test_existing_bootstrap_accounts_are_accepted(self): + for user in ['root', 'adam', 'debian', 'bob', 'cloud-admin', '_admin', 'Admin.1']: + with self.subTest(user=user): + self.assertEqual(destination(user + '@example.com'), (user, 'example.com', None)) + self.assertEqual(destination('debian@[2001:db8::1]', 22), ('debian', '2001:db8::1', 22)) + + def test_nonroot_bootstrap_uses_noninteractive_sudo_and_preserves_stdin(self): + calls = [] + def runner(args, **kwargs): + calls.append((args, kwargs)) + return subprocess.CompletedProcess(args, 0, b'root verified', b'') + ssh = SSH('debian@host', runner=runner) + self.assertEqual(ssh.run('id -u', b'opaque input').stdout, b'root verified') + args, kwargs = calls[0] + self.assertEqual(args[args.index('-l') + 1], 'debian') + self.assertEqual(shlex.split(args[-1]), ['sudo', '-n', '--', 'sh', '-c', 'id -u']) + self.assertEqual(kwargs['input'], b'opaque input') + ssh.run('printf %s "$SSH_CONNECTION"', privileged=False) + self.assertEqual(shlex.split(calls[1][0][-1])[:2], ['sh', '-c']) + + def test_sudo_cleared_connection_environment_uses_unprivileged_fallback(self): + class PreflightComplete(Exception): + pass + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'user-data' + source.write_bytes(b'opaque input') + release = root / 'os-release' + release.write_text('ID=debian\nVERSION_ID=13\n') + systemd_run = root / 'systemd-run' + systemd_run.write_text('#!/bin/sh\nexit 0\n') + systemd_run.chmod(0o755) + environment = dict(os.environ, PATH=str(root) + os.pathsep + os.environ['PATH']) + environment.pop('SSH_CONNECTION', None) + fallbacks = [] + def runner(args, **kwargs): + if '-G' in args: + return subprocess.CompletedProcess(args, 0, 'hostname 192.0.2.2\nport 2222\n', '') + command = shlex.split(args[-1]) + script = command[-1] + if 'ssh_host_' in script: + self.assertEqual(command[:3], ['sudo', '-n', '--']) + tools = ('id() { printf 0; }; dpkg() { printf amd64; }; ' + "cat() { printf 'ssh-ed25519 AAAA fixture\\n'; }; ") + script = script.replace('/etc/os-release', shlex.quote(str(release))) + result = subprocess.run(['sh', '-c', tools + script], capture_output=True, env=environment) + self.assertEqual(result.returncode, 0, result.stderr) + return result + if 'SSH_CONNECTION' in script: + self.assertEqual(command[:2], ['sh', '-c']) + fallbacks.append(script) + return subprocess.run(command, capture_output=True, + env=dict(environment, SSH_CONNECTION='192.0.2.1 50000 192.0.2.2 2222')) + self.assertIn('/var/lib/apex-init/remote.json', script) + self.assertIn('ssh-ed25519 AAAA', transport.trust_file.read_text()) + raise PreflightComplete + transport = SSH('adam@example.com', runner=runner) + with patch('engine.provisioning.transport.SSH', return_value=transport), \ + patch('engine.provisioning.transport.build_bundle', return_value=(b'bundle', 'a' * 64, 'b' * 40)), \ + patch('engine.provisioning.transport.shutil.which', return_value='/usr/bin/ssh'): + with self.assertRaises(PreflightComplete): + initialize(root, 'adam@example.com', source, progress=lambda message: None) + self.assertEqual(len(fallbacks), 1) + + def test_remote_script_quoting_transports_literal_values(self): + received = [] + def runner(args, **kwargs): + command = shlex.split(args[-1]) + self.assertEqual(command[:2], ['sh', '-c']) + result = subprocess.run(command, input=kwargs.get('input'), capture_output=True) + received.append(result.stdout) + return result + value = "spaces '$HOME' `echo unsafe`; $(echo unsafe)" + SSH('root@host', runner=runner).run('printf %s ' + shlex.quote(value)) + self.assertEqual(received, [value.encode()]) + + def test_secret_bytes_go_over_stdin_and_diagnostics_are_redacted(self): + secret = b'private key material' + def runner(args, **kwargs): + self.assertEqual(kwargs['input'], secret) + self.assertNotIn(secret.decode(), repr(args)) + return subprocess.CompletedProcess(args, 1, b'', secret) + with self.assertRaises(TransportError) as caught: + SSH('root@host', runner=runner).run('cat >/dev/null', secret) + self.assertNotIn(secret.decode(), str(caught.exception)) + + def test_fresh_login_disables_mux_password_and_pins_verified_identity(self): + calls = [] + def runner(args, **kwargs): + calls.append(args) + return subprocess.CompletedProcess(args, 0, b'', b'') + ssh = SSH('root@original-alias', runner=runner) + ssh.resolved_host = '192.0.2.42' + ssh.trust_file = Path('/tmp/trusted') + ssh.fresh_login() + args = calls[0] + self.assertIn('-T', args) + for option in ('ControlMaster=no', 'ControlPath=none', 'BatchMode=yes', + 'PreferredAuthentications=publickey', 'StrictHostKeyChecking=yes', + 'Hostname=192.0.2.42', 'HostKeyAlias=apex-init-verified-target'): + self.assertIn(option, args) + self.assertEqual(args[args.index('-l') + 1], 'adam') + self.assertEqual(args[args.index('-p') + 1], '2222') + + def test_failed_first_fresh_login_rolls_back_without_finalizing(self): + calls = [] + class FailedSSH: + def fresh_login(self): + raise TransportError('key rejected') + def invoke(operation, *args): + calls.append(operation) + return {'status': 'transition'} + with self.assertRaises(TransportError): + protected_handover(FailedSSH(), invoke, 'a' * 32, 22, '192.0.2.1') + self.assertEqual(calls, ['begin', 'rollback']) + + def test_failed_second_fresh_login_never_commits(self): + calls = [] + class SecondFailure: + count = 0 + def fresh_login(self): + self.count += 1 + if self.count == 2: + raise TransportError('final policy rejected login') + def invoke(operation, *args): + calls.append(operation) + return {'status': 'transition'} + with self.assertRaises(TransportError): + protected_handover(SecondFailure(), invoke, 'a' * 32, 22, '192.0.2.1') + self.assertEqual(calls, ['begin', 'finalize', 'rollback']) + + def test_two_successful_fresh_logins_precede_commit(self): + calls = [] + class VerifiedSSH: + def fresh_login(self): + calls.append('fresh') + def invoke(operation, *args): + calls.append(operation) + return {'status': 'transition'} + protected_handover(VerifiedSSH(), invoke, 'a' * 32, 22, '192.0.2.1') + self.assertEqual(calls, ['begin', 'fresh', 'finalize', 'fresh', 'commit']) + + def test_lost_commit_reply_is_reconciled_without_rolling_back_committed_access(self): + from engine.provisioning.access import Transition + from tests.test_provision_access import FakeHost + with tempfile.TemporaryDirectory() as directory: + host = FakeHost(Path(directory)) + transition = Transition(host, now=lambda: 1000, security=host.security) + calls = [] + class VerifiedSSH: + def fresh_login(self): + pass + def invoke(operation, token, port, address, fresh): + calls.append(operation) + if operation == 'status': + self.assertTrue(fresh) + return transition.load() + result = transition.execute(operation, token, port, address) + if operation == 'commit': + raise TransportError('SSH reply lost') + return result + protected_handover(VerifiedSSH(), invoke, 'a' * 32, 22, '192.0.2.1') + self.assertEqual(transition.load()['status'], 'committed') + self.assertEqual(host.ports, [2222]) + self.assertFalse(host.armed) + self.assertEqual(host.restored, 0) + self.assertEqual(calls, ['begin', 'finalize', 'commit', 'status']) + + def test_known_uncommitted_lost_reply_still_rolls_back(self): + calls = [] + class VerifiedSSH: + def fresh_login(self): + pass + def invoke(operation, *args): + calls.append(operation) + if operation == 'commit': + raise TransportError('SSH failed before commit') + return {'status': 'final' if operation == 'status' else 'transition', 'token': 'a' * 32} + with self.assertRaises(TransportError): + protected_handover(VerifiedSSH(), invoke, 'a' * 32, 22, '192.0.2.1') + self.assertEqual(calls, ['begin', 'finalize', 'commit', 'status', 'rollback']) + + def test_lost_commit_reply_reports_verified_or_uncertain_resume_endpoints(self): + from engine.provisioning.access import Transition + from tests.test_provision_access import FakeHost + for unreachable in (False, True): + with self.subTest(unreachable=unreachable), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'user-data' + source.write_bytes(b'opaque input') + host = FakeHost(root) + transition = Transition(host, now=lambda: 1000, security=host.security) + messages, commands = [], [] + def runner(args, **kwargs): + if '-G' in args: + return subprocess.CompletedProcess(args, 0, 'hostname 192.0.2.2\nport 22\n', '') + command = shlex.split(args[-1]) + script = command[-1] + commands.append(script) + value = b'' + if 'ssh_host_' in script: + value = b'192.0.2.1 50000 192.0.2.2 22\nssh-ed25519 AAAA fixture\n' + elif 'mktemp -d /var/tmp/apex-init.' in script: + value = b'/var/tmp/apex-init.0123456789\n' + elif script.startswith('if test -f /var/tmp'): + value = b'0' + elif script.startswith('cd ') and 'engine.provisioning.access' in script: + access_args = shlex.split(script)[3:] + operation = access_args[3] + token = access_args[access_args.index('--token') + 1] + port = int(access_args[access_args.index('--original-port') + 1]) if '--original-port' in access_args else None + address = access_args[access_args.index('--client-address') + 1] if port else None + value = json.dumps(transition.execute(operation, token, port, address)).encode() + if operation == 'commit': + return subprocess.CompletedProcess(args, 255, b'', b'connection lost') + elif 'cat /var/lib/apex-init/access/state.json' in script: + self.assertEqual(args[args.index('-l') + 1], 'adam') + self.assertEqual(args[args.index('-p') + 1], '2222') + self.assertEqual(command[:3], ['sudo', '-n', '--']) + if unreachable: + return subprocess.CompletedProcess(args, 255, b'', b'connection lost') + value = json.dumps(transition.load()).encode() + elif 'engine.provisioning.target complete' in script: + value = b'{"status": "complete"}' + return subprocess.CompletedProcess(args, 0, value, b'') + def transport(*args): + return SSH(*args, runner=runner) + with patch('engine.provisioning.transport.SSH', side_effect=transport), \ + patch('engine.provisioning.transport.build_bundle', return_value=(b'bundle', 'a' * 64, 'b' * 40)), \ + patch('engine.provisioning.transport.shutil.which', return_value='/usr/bin/ssh'): + if unreachable: + with self.assertRaises(HandoverUncertain): + initialize(root, 'debian@example.com', source, progress=messages.append) + self.assertTrue(any('outcome is unknown' in message for message in messages)) + self.assertIn('adam@example.com --port 2222', messages[-2]) + self.assertIn('debian@example.com --port 22', messages[-1]) + self.assertFalse(any('Safe resume' in message or 'pending rollback' in message for message in messages)) + else: + self.assertEqual(initialize(root, 'debian@example.com', source, progress=messages.append), {'status': 'complete'}) + self.assertIn('Safe resume: ./apex init adam@example.com --port 2222', messages[-1]) + self.assertEqual(transition.load()['status'], 'committed') + self.assertFalse(host.armed) + self.assertFalse(any('access rollback' in script for script in commands)) + + def test_bundle_excludes_repository_and_environment_secrets(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'engine/provisioning').mkdir(parents=True) + (root / 'engine/__init__.py').write_text('') + (root / 'engine/provisioning/seed.sh').write_text('#!/bin/sh\n') + (root / '.env').write_text('private secret') + (root / 'engine/private.env').write_text('private secret') + data, digest, revision = build_bundle(root) + with tarfile.open(fileobj=io.BytesIO(data)) as archive: + self.assertEqual(set(archive.getnames()), {'engine/__init__.py', 'engine/provisioning/seed.sh'}) + self.assertNotIn(b'private secret', data) + self.assertEqual(build_bundle(root)[1], digest) + + def test_missing_input_is_a_safe_validation_error(self): + with self.assertRaises(TransportError): + local_file('/nonexistent/apex-user-data', 'User-data') + + def test_init_from_node_checkout_does_not_resolve_local_identity(self): + from engine import cli + root = Path(__file__).resolve().parents[1] + with patch.object(cli, '_resolve_layout', return_value=(str(root), str(root), None, True)), \ + patch.object(cli, 'resolve', side_effect=AssertionError('local identity used')): + with self.assertRaises(SystemExit) as caught: + cli.main(['init', '--help']) + self.assertEqual(caught.exception.code, 0) + + def test_dead_enrollment_unit_stops_polling_and_removes_input(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'user-data' + source.write_bytes(b'opaque input') + calls = [] + class Remote: + user, host, identity = 'root', 'example.com', None + def __init__(self, *args): + self.resolved_port = 22 + def resolve(self): + pass + def run(self, script, data=None, **kwargs): + calls.append(script) + if 'ssh_host_' in script: + value = b'192.0.2.1 50000 192.0.2.2 22\nssh-ed25519 AAAA fixture\n' + elif 'mktemp -d /var/tmp/apex-init.' in script: + value = b'/var/tmp/apex-init.0123456789\n' + elif script.startswith('if test -f /var/tmp'): + value = b'stopped' + else: + value = b'' + return subprocess.CompletedProcess([], 0, value, b'') + messages = [] + with patch('engine.provisioning.transport.SSH', Remote), \ + patch('engine.provisioning.transport.build_bundle', return_value=(b'bundle', 'a' * 64, 'b' * 40)), \ + patch('engine.provisioning.transport.shutil.which', return_value='/usr/bin/ssh'): + with self.assertRaises(TransportError) as caught: + initialize(root, 'root@example.com', source, progress=messages.append) + self.assertIn('/runner.log', str(caught.exception)) + self.assertTrue(any(script.startswith('rm -f ') and '/user-data ' in script for script in calls)) + self.assertIn('root@example.com --port 22', messages[-1]) + + def test_completion_failure_keeps_administrator_resume_endpoint(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'user-data' + source.write_bytes(b'opaque input') + class Remote: + user, host, identity = 'root', 'example.com', None + def __init__(self, *args): + self.resolved_port = 22 + def resolve(self): + pass + def run(self, script, data=None, **kwargs): + if 'ssh_host_' in script: + value = b'192.0.2.1 50000 192.0.2.2 22\nssh-ed25519 AAAA fixture\n' + elif 'mktemp -d /var/tmp/apex-init.' in script: + value = b'/var/tmp/apex-init.0123456789\n' + elif script.startswith('if test -f /var/tmp'): + value = b'0' + elif 'engine.provisioning.target complete' in script: + raise TransportError('record failed') + else: + value = b'' + return subprocess.CompletedProcess([], 0, value, b'') + messages = [] + with patch('engine.provisioning.transport.SSH', Remote), \ + patch('engine.provisioning.transport.build_bundle', return_value=(b'bundle', 'a' * 64, 'b' * 40)), \ + patch('engine.provisioning.transport.protected_handover'), \ + patch('engine.provisioning.transport.shutil.which', return_value='/usr/bin/ssh'): + with self.assertRaises(TransportError): + initialize(root, 'root@example.com', source, progress=messages.append) + self.assertIn('adam@example.com --port 2222', messages[-1]) + + +class TargetPathTests(unittest.TestCase): + """Run the real shell guards with only Debian stat/mv translated on macOS.""" + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.bin = self.root / 'bin' + self.bin.mkdir() + (self.bin / 'stat').write_text('#!' + sys.executable + '\n' + ''' +import os, stat, sys +path = sys.argv[-1] +info = os.lstat(path) +field = sys.argv[sys.argv.index('-c') + 1] +values = {'%a': format(stat.S_IMODE(info.st_mode), 'o'), '%h': str(info.st_nlink), + '%u': '1000' if path == os.environ.get('TEST_NONROOT_PATH') else '0'} +print(values[field]) +''') + (self.bin / 'mv').write_text('#!' + sys.executable + '\nimport os, sys\nos.replace(sys.argv[-2], sys.argv[-1])\n') + for command in self.bin.iterdir(): + command.chmod(0o755) + self.environment = dict(os.environ, PATH=str(self.bin) + os.pathsep + os.environ['PATH']) + + def write(self, path, data=b'new metadata'): + return subprocess.run(['sh', '-c', remote_atomic_input(str(path))], input=data, + capture_output=True, env=self.environment) + + def test_atomic_write_ignores_predictable_old_temporary_symlink(self): + destination = self.root / 'remote.json' + victim = self.root / 'victim' + victim.write_bytes(b'preserve me') + destination.with_name('remote.json.new').symlink_to(victim) + result = self.write(destination) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(destination.read_bytes(), b'new metadata') + self.assertEqual(victim.read_bytes(), b'preserve me') + self.assertEqual(destination.stat().st_mode & 0o777, 0o600) + self.assertFalse(list(self.root.glob('.apex-write.*'))) + + def test_destination_symlink_is_rejected_without_touching_victim(self): + victim = self.root / 'victim' + victim.write_bytes(b'preserve me') + destination = self.root / 'remote.json' + destination.symlink_to(victim) + result = self.write(destination) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(victim.read_bytes(), b'preserve me') + + def test_redirected_parent_is_rejected_before_creating_files(self): + victim = self.root / 'victim' + victim.mkdir() + redirected = self.root / 'apex-init' + redirected.symlink_to(victim, target_is_directory=True) + result = self.write(redirected / 'remote.json') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(list(victim.iterdir()), []) + + def test_unowned_parent_is_rejected_before_creating_files(self): + state = self.root / 'apex-init' + state.mkdir() + self.environment['TEST_NONROOT_PATH'] = str(state) + result = self.write(state / 'remote.json') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(list(state.iterdir()), []) + + def test_writable_ancestor_is_rejected_before_creating_files(self): + state = self.root / 'apex-init' + state.mkdir(mode=0o777) + state.chmod(0o777) + result = self.write(state / 'remote.json') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(list(state.iterdir()), []) + + def test_hardlinked_destination_is_rejected_without_touching_victim(self): + victim = self.root / 'victim' + victim.write_bytes(b'preserve me') + destination = self.root / 'remote.json' + os.link(victim, destination) + result = self.write(destination) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(victim.read_bytes(), b'preserve me') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_provision_userdata.py b/tests/test_provision_userdata.py new file mode 100644 index 0000000..00f8e48 --- /dev/null +++ b/tests/test_provision_userdata.py @@ -0,0 +1,30 @@ +import importlib.util +import unittest + +from engine.provisioning.contract import ContractError +from engine.provisioning.userdata import decode_user_data + + +@unittest.skipUnless(importlib.util.find_spec('yaml'), 'target-only python3-yaml is unavailable') +class UserDataTests(unittest.TestCase): + def test_duplicate_mapping_keys_rejected_before_normalization(self): + with self.assertRaisesRegex(ContractError, 'unique'): + decode_user_data(b'hostname: one.example.com\nhostname: two.example.com\n') + + def test_alias_and_unsafe_yaml_do_not_execute_or_echo_content(self): + for content in (b'a: &a [*a]', b'!!python/object/apply:os.system [SECRET_SENTINEL]', + b'a: [SECRET_SENTINEL'): + with self.subTest(content=content): + with self.assertRaises(ContractError) as error: + decode_user_data(content) + self.assertNotIn('SECRET_SENTINEL', str(error.exception)) + + def test_normal_supported_document(self): + import yaml + from tests.test_provision_contract import example + spec = decode_user_data(yaml.safe_dump(example()).encode()) + self.assertEqual(spec.hostname, 'metis.a1.example.com') + + def test_yaml_boolean_mapping_key_rejected(self): + with self.assertRaisesRegex(ContractError, 'unique strings'): + decode_user_data(b'true: value\n')