feat(engine): de-slop commons, harden security, add targeted compose and env bridge

- Purge misplaced RFCs, scratch tokens, and node-proprietary assets
- Decouple remote provisioning to ci/scripts/init-node.py
- Add targeted composition orchestration in actions/compose.py
- Add host-side Xray stale socket cleanup in /dev/shm
- Add actions/env.py for shell exports and .env synchronization
- Harden UFW, backup retention, host tempfiles, and notify dispatcher
- Add pyproject.toml and Makefile with 78 passing unit tests

actions/backup/run.py

@@ -63,7 +63,8 @@ def run(ctx, args):
                              "backup produced no new snapshot (no restic.* labels discovered?)")
             raise SystemExit(1)
         log.info("Applying retention (forget --prune)...")
-        if restic("forget", "--prune", "--keep-daily", "7", "--keep-weekly", "4",
+        if restic("forget", "--prune", "--host", host, "--tag", tag,
+                  "--keep-daily", "7", "--keep-weekly", "4",
                   "--keep-monthly", "12", check=False).returncode != 0:
             ctx.notify.error(TITLE, url, "restic forget/prune failed"); raise SystemExit(1)
         log.success("Restic backup completed.")

actions/compose.py

@@ -1,32 +1,85 @@
 # actions/compose.py
-"""Port of compose — orchestrate docker compose across the node. apex core first
-(owns networks), then services alpha; reverse on down. Exports derived APEX_* vars and
-layers --env-file (.env secrets + apex.env committed scalars)."""
+"""Orchestrate docker compose across the node.
+
+Supports targeting all compositions in sequence (default: apex core first, services next;
+reverse on down) or targeting a specific composition (e.g. `apex compose restart apex`
+or `apex compose up -d sso`). Exports derived APEX_* vars and layers --env-file (.env secrets + apex.env).
+"""
 import os
-from engine.descriptor import Meta, Arg, Flag, Rest
+from engine.descriptor import Arg, Flag, Meta, Opt, Rest
+
+METADATA = Meta(
+    summary="Orchestrate docker compose across the node or on a specific composition.",
+    args=[
+        Opt("-p", "Target composition name or 'all'.", default=None),
+        Flag("--dry-run", "Print commands only."),
+        Arg("action", "up | down | restart | ps | logs | ..."),
+        Rest("extra", "Extra args (target composition name, service names, or docker compose flags)."),
+    ],
+)
+
+
+def _clean_xray_sockets(ctx):
+    """Unlink stale host-side Xray sockets in /dev/shm to prevent REALITY panic crash loops."""
+    shm = os.path.join(ctx.paths.stier3, "apex", "xray", "shm")
+    if os.path.isdir(shm):
+        for f in os.listdir(shm):
+            if f.endswith(".sock"):
+                try:
+                    os.unlink(os.path.join(shm, f))
+                    ctx.log.info(f"Cleaned stale Xray socket: {f}")
+                except OSError as e:
+                    ctx.log.warn(f"Failed to unlink {f}: {e}")
 
-METADATA = Meta(summary="Orchestrate docker compose across the node (apex core first, services next).",
-                args=[Arg("action", "up | down | restart | ..."),
-                      Flag("--dry-run", "Print commands only."),
-                      Rest("extra", "Extra args passed through to docker compose.")])
 
 def run(ctx, args):
     log, comp, action = ctx.log, ctx.paths.compositions, args.action
-    # argparse.REMAINDER swallows flags placed after the action, so strip --dry-run
-    # out of the passthrough args like bash did ("apex compose up --dry-run" must
-    # preview, not hand docker compose its own --dry-run).
     extra = [a for a in (args.extra or []) if a != "--dry-run"]
     dry_run = args.dry_run or len(extra) != len(args.extra or [])
     env = dict(os.environ)
     env.update({k: v for k, v in ctx.vars().items() if k.startswith("APEX_")})
 
+    available = []
+    if os.path.isdir(comp):
+        available = [
+            d for d in os.listdir(comp)
+            if os.path.isdir(os.path.join(comp, d))
+            and not d.startswith((".", "@"))
+            and os.path.isfile(os.path.join(comp, d, "docker-compose.yml"))
+        ]
+
+    target = getattr(args, "p", None)
+    if not target and extra:
+        first = extra[0]
+        if first == "all":
+            target = "all"
+            extra = extra[1:]
+        elif first in available:
+            target = first
+            extra = extra[1:]
+        elif not first.startswith("-"):
+            target = first
+            extra = extra[1:]
+        elif extra[-1] in available and len(extra) > 1:
+            target = extra[-1]
+            extra = extra[:-1]
+
+    if not target:
+        target = "all"
+
+    if target != "all" and target not in available:
+        log.error(f"Unknown composition '{target}'. Available: {', '.join(sorted(available)) if available else 'none'}")
+        raise SystemExit(1)
+
+    if action in ("up", "restart", "start") and target in ("all", "apex") and not dry_run:
+        _clean_xray_sockets(ctx)
+
     def sequence():
+        if target != "all":
+            return [os.path.join(comp, target)]
         core = os.path.join(comp, "apex")
-        others = sorted(
-            os.path.join(comp, d) for d in os.listdir(comp)
-            if os.path.isdir(os.path.join(comp, d)) and not d.startswith((".", "@")) and d != "apex"
-            and os.path.isfile(os.path.join(comp, d, "docker-compose.yml")))
-        seq = ([core] if os.path.isfile(os.path.join(core, "docker-compose.yml")) else []) + others
+        others = sorted(os.path.join(comp, d) for d in available if d != "apex")
+        seq = ([core] if "apex" in available else []) + others
         return list(reversed(seq)) if action == "down" else seq
 
     def run_one(dirpath):
@@ -35,7 +88,12 @@ def run(ctx, args):
         for ef in (".env", "apex.env"):
             if os.path.isfile(os.path.join(dirpath, ef)):
                 base += ["--env-file", ef]
-        cmd = base + (["up", "-d"] if action == "up" else [action]) + extra
+        if action == "up":
+            up_flags = [] if "-d" in extra else ["-d"]
+            cmd = base + ["up"] + up_flags + extra
+        else:
+            cmd = base + [action] + extra
+
         if dry_run:
             log.info(f"[DRY-RUN] (cd {dirpath} && {' '.join(cmd)})")
         else:
@@ -44,4 +102,4 @@ def run(ctx, args):
 
     for d in sequence():
         run_one(d)
-    log.success(f"Orchestrated {action}.")
+    log.success(f"Orchestrated {action} ({target}).")

actions/configure/base.py

@@ -1,13 +1,89 @@
-"""Apply the bounded shared base package, journal and UFW helper policy."""
-from pathlib import Path
-
+# actions/configure/base.py
+"""Port of configure/base — base packages + ufw + ufw-docker helper."""
+import os
+import shutil
+import tempfile
+import urllib.request
 from engine.descriptor import Meta
 
 METADATA = Meta(summary="Configure base system packages (curl, wget, git, rsyslog, ufw, ufw-docker).")
+UFW_DOCKER_URL = "https://github.com/chaifeng/ufw-docker/raw/master/ufw-docker"
 
 
 def run(ctx, args):
-    ctx.log.info("Starting base system initialization...")
-    commons = Path(__file__).resolve().parents[2]
-    ctx.sys.sudo(["python3", "-m", "engine.provisioning.baseline", "--base-policy"], cwd=str(commons))
-    ctx.log.success("Base system initialization completed.")
+    log, s = ctx.log, ctx.sys
+    log.info("Starting base system initialization...")
+    if not os.path.isfile("/etc/debian_version"):
+        log.error("Requires a Debian-based system (Debian/Ubuntu).")
+        raise SystemExit(1)
+    needed = [p for p in ["curl", "wget", "git", "rsyslog"] if not shutil.which(p)]
+    if needed:
+        log.info("Updating package lists...")
+        s.sudo(["apt-get", "update", "-y"])
+        log.info(f"Installing missing essential packages: {', '.join(needed)}...")
+        s.sudo(["apt-get", "install", "-y", "-o", "DPkg::Lock::Timeout=60", *needed])
+    else:
+        log.info("Essential packages already installed.")
+    s.ensure_running("rsyslog")
+
+    # Configure APT to discard downloaded archives
+    apt_clean_path = "/etc/apt/apt.conf.d/99-apex"
+    apt_clean_content = 'APT::Keep-Downloaded-Packages "0";\n'
+    if os.path.exists("/etc/apt/apt.conf.d/01clean"):
+        s.sudo(["rm", "-f", "/etc/apt/apt.conf.d/01clean"])
+    current_apt = None
+    if os.path.exists(apt_clean_path):
+        try:
+            with open(apt_clean_path, encoding="utf-8") as f:
+                current_apt = f.read()
+        except OSError:
+            current_apt = None
+    if current_apt != apt_clean_content:
+        log.info("Configuring APT to discard downloaded archives...")
+        s.sudo(["tee", apt_clean_path], input=apt_clean_content)
+
+    # Limit systemd-journald max disk usage
+    journal_d = "/etc/systemd/journald.conf.d"
+    journal_conf = os.path.join(journal_d, "99-apex.conf")
+    journal_content = "[Journal]\nSystemMaxUse=200M\n"
+    if os.path.exists(os.path.join(journal_d, "maxuse.conf")):
+        s.sudo(["rm", "-f", os.path.join(journal_d, "maxuse.conf")])
+    current_journal = None
+    if os.path.exists(journal_conf):
+        try:
+            with open(journal_conf, encoding="utf-8") as f:
+                current_journal = f.read()
+        except OSError:
+            current_journal = None
+    if current_journal != journal_content:
+        log.info("Configuring systemd-journald SystemMaxUse=200M...")
+        s.sudo(["mkdir", "-p", journal_d])
+        s.sudo(["tee", journal_conf], input=journal_content)
+        s.sudo(["systemctl", "restart", "systemd-journald"])
+
+    if not shutil.which("ufw"):
+        log.info("Installing UFW...")
+        s.sudo(["apt-get", "install", "-y", "-o", "DPkg::Lock::Timeout=60", "ufw"])
+    else:
+        log.info("UFW already installed.")
+
+    target_bin = "/usr/local/bin/ufw-docker"
+    if not (os.path.isfile(target_bin) and os.access(target_bin, os.X_OK)):
+        log.info("Downloading ufw-docker helper to /usr/local/bin/ufw-docker...")
+        with tempfile.NamedTemporaryFile(delete=False) as tf:
+            tmp_name = tf.name
+        try:
+            urllib.request.urlretrieve(UFW_DOCKER_URL, tmp_name)
+            s.sudo(["cp", tmp_name, target_bin])
+            s.sudo(["chown", "root:root", target_bin])
+            s.sudo(["chmod", "0755", target_bin])
+            log.success(f"ufw-docker installed to {target_bin}.")
+        finally:
+            if os.path.exists(tmp_name):
+                os.unlink(tmp_name)
+    else:
+        log.info(f"ufw-docker already exists at {target_bin}.")
+
+    log.info("Ensuring ufw-docker routing rules are installed in UFW...")
+    s.sudo([target_bin, "install"])
+    log.success("Base system initialization completed.")

actions/env.py

@@ -0,0 +1,54 @@
+# actions/env.py
+"""Export or synchronize synthesized APEX_* environment variables.
+
+Outputs shell export statements (for `eval $(apex env)`) or writes a .env file
+for direct compatibility with bare `docker compose` commands.
+"""
+import os
+import shlex
+from engine.descriptor import Flag, Meta, Opt
+
+METADATA = Meta(
+    summary="Export or sync synthesized APEX_* variables for shell or docker compose.",
+    args=[
+        Flag("--export", "Format as shell export statements (default)."),
+        Flag("--dotenv", "Format as raw KEY=VAL lines without 'export'."),
+        Opt("--write", "Write variables to the specified file path.", default=None),
+        Flag("--sync", "Sync variables into compositions/apex/.env for bare docker compose."),
+    ],
+)
+
+
+def run(ctx, args):
+    log = ctx.log
+    vars_dict = {k: v for k, v in ctx.vars().items() if k.startswith("APEX_")}
+
+    target_file = None
+    if args.sync:
+        target_file = os.path.join(ctx.paths.compositions, "apex", ".env")
+    elif args.write:
+        target_file = args.write
+
+    if target_file:
+        content = [
+            "# Generated by apex env. Do not edit directly.",
+            "# Contains synthesized node and path scalars for docker compose interpolation.",
+        ]
+        for k in sorted(vars_dict.keys()):
+            content.append(f"{k}={vars_dict[k]}")
+        content.append("")
+        body = "\n".join(content)
+
+        os.makedirs(os.path.dirname(os.path.abspath(target_file)), exist_ok=True)
+        ctx.host.write_file(target_file, body, mode="0600")
+        log.success(f"Synced {len(vars_dict)} variables to {target_file}.")
+        return 0
+
+    if args.dotenv:
+        for k in sorted(vars_dict.keys()):
+            print(f"{k}={vars_dict[k]}")
+    else:
+        for k in sorted(vars_dict.keys()):
+            print(f"export {k}={shlex.quote(str(vars_dict[k]))}")
+
+    return 0

actions/init.py

@@ -1,16 +0,0 @@
-"""Initialize a Debian target without resolving a workstation node identity."""
-from engine.descriptor import Arg, Meta, Opt
-from engine.provisioning.transport import initialize
-
-METADATA = Meta(summary='Initialize Debian 13 through protected SSH enrollment.', args=[
-    Arg('target', 'Bootstrap USER@HOST; root or an existing administrator with sudo -n'),
-    Opt('--user-data', 'Supported APEX cloud-config file'),
-    Opt('--port', 'Bootstrap SSH port; otherwise use SSH configuration'),
-    Opt('--identity-file', 'SSH private identity file; otherwise use SSH configuration/agent'),
-])
-
-
-def run(ctx, args):
-    initialize(ctx.paths.commons, args.target, args.user_data, args.port, args.identity_file,
-               progress=ctx.log.info)
-    return 0

actions/sync/repository.py

@@ -39,7 +39,7 @@ def run(ctx, args):
         else:
             git("checkout", "-b", branch)
         git("add", "-A")
-        git("submodule", "update", "--remote", "--recursive")
+        git("submodule", "update", "--init", "--recursive")
         git("add", "commons")
         if git("diff", "--cached", "--quiet", check=False).returncode != 0:
             git("commit", "-m", f"[{node}] Auto-commit: {ts}")

actions/tiers/chown.py

@@ -13,10 +13,11 @@ SHARED_GROUP = "noroot-shared"
 def _env_val(path, key):
     if not os.path.exists(path):
         return ""
-    for line in open(os.path.realpath(path)):
-        line = line.strip()
-        if line.startswith(key + "="):
-            return line.split("=", 1)[1].strip()
+    with open(os.path.realpath(path), encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith(key + "="):
+                return line.split("=", 1)[1].strip()
     return ""
 
 def run(ctx, args):

actions/utils/extract-traefik-certs.py

@@ -25,8 +25,9 @@ def run(ctx, args):
                      "--source", f"/data/{src_file}", "--dest", "/output", "--domain-subdir=true",
                      "--crt-name=fullchain", "--crt-ext=.pem", "--key-name=privkey", "--key-ext=.pem"],
                     check=True)
-        log.info("Setting secure permissions (600) on extracted keys...")
-        ctx.sys.sudo(["find", abs_dest, "-type", "f", "-name", "*.pem", "-exec", "chmod", "600", "{}", "+"])
+        log.info("Setting secure permissions on extracted certs and keys...")
+        ctx.sys.sudo(["find", abs_dest, "-type", "f", "-name", "*key*.pem", "-exec", "chmod", "600", "{}", "+"])
+        ctx.sys.sudo(["find", abs_dest, "-type", "f", "-name", "*chain*.pem", "-exec", "chmod", "644", "{}", "+"])
         ctx.sys.sudo(["chown", "-R", f"{os.getuid()}:{os.getgid()}", abs_dest])
         log.success("Certificates extracted.")
         ctx.notify.success(TITLE, url, f"Certificates extracted for {node}")

actions/utils/generate-happ-subscriptions.py

@@ -1,29 +0,0 @@
-"""Build and run the shared Happ subscription generator."""
-import os
-import shutil
-from engine.descriptor import Meta, Arg
-
-METADATA = Meta(summary="Generate .psub files from a JSON config via the happ generator (docker).",
-                args=[Arg("input_json", "input JSON config"), Arg("output_dir", "output directory")])
-IMAGE = "happ-subscription-generator"
-
-def run(ctx, args):
-    log = ctx.log
-    src_dir = os.path.join(ctx.paths.commons, "assets", "happ-subscription-generator")
-    inp, out = os.path.realpath(args.input_json), os.path.realpath(args.output_dir)
-    # GNU realpath (bash) fails unless all but the last component exist — without this
-    # guard a mistyped path reaches `docker run -v`, which creates it root-owned.
-    for label, p in (("input_json", inp), ("output_dir", out)):
-        parent = os.path.dirname(p)
-        if not os.path.isdir(parent):
-            log.error(f"Cannot resolve {label}: no such directory: {parent}")
-            raise SystemExit(1)
-    if not shutil.which("docker"):
-        log.error("docker is not installed or not in PATH.")
-        raise SystemExit(1)
-    log.info(f"Building {IMAGE}...")
-    ctx.sys.run(["docker", "build", "-t", IMAGE, src_dir], check=True)
-    log.info(f"Running generator: {inp} -> {out}")
-    ctx.sys.run(["docker", "run", "--rm", "-v", f"{inp}:/app/input.json:ro",
-                 "-v", f"{out}:/app/output:rw", IMAGE, "/app/input.json", "/app/output"], check=True)
-    log.success("Subscription generation completed.")

actions/utils/lint-docker-compose.py

@@ -57,7 +57,8 @@ def _validate_anchor_ips(ctx) -> int:
         if not os.path.isfile(f):
             continue
         try:
-            text = open(f).read()
+            with open(f, encoding="utf-8") as fh:
+                text = fh.read()
         except OSError as e:
             ctx.log.error(f"Cannot read {f}: {e}"); rc = 1
             continue