refactor: replace loki store with Grafana Alloy push agent

An empty .apex-root-owned marker in a composition dir makes tiers/useradd
skip its noroot user + .env UID injection, and tiers/chown force its tier
data (dir + children, recursively) to root:root instead of noroot-<proj>.
For services that run as root and reject non-root-owned data (e.g. seafile-mc).

README.md

@@ -3,6 +3,8 @@
 The **commons** of the apex fleet: a stock-python engine (`apex <group>/<action>`) and
 the shared core docker-compose, consumed by every node as the `commons/` git submodule.
 
+Full documentation: [docs.ermnvldmr.com/en/apex](https://docs.ermnvldmr.com/en/apex/).
+
 ## Layout
 - `apex` — launcher (python3, stdlib only); node `init.sh` symlinks it onto `PATH`.
 - `engine/` — the runner: CLI, identity, overlay, context, and the `ctx.*` helper library.

actions/tiers/chown.py

@@ -46,6 +46,16 @@ def run(ctx, args):
         full = os.path.join(comp, proj)
         if not os.path.isdir(full) or proj.startswith(".") or proj.startswith("@"):
             continue
+        # opt-out: .apex-root-owned forces this project's tier data (dir + children,
+        # recursively) to root:root — for services that run as root and reject
+        # non-root-owned data (e.g. seafile-mc). Blocks noroot ownership entirely.
+        if os.path.isfile(os.path.join(full, ".apex-root-owned")):
+            log.info(f"Applying root ownership for {proj} (.apex-root-owned)")
+            for tp in tiers:
+                for d in (os.path.join(tp, proj), os.path.join(tp, "shared", proj)):
+                    if os.path.isdir(d):
+                        s.sudo(["chown", "-R", "root:root", d]); log.success(f"  Owned {d} (root)")
+            continue
         env_path = os.path.join(full, ".env")
         uid = _env_val(env_path, "APEX_UID")
         gid = _env_val(env_path, "APEX_GID")

actions/tiers/useradd.py

@@ -34,6 +34,12 @@ def run(ctx, args):
         full = os.path.join(comp, proj)
         if not os.path.isdir(full) or proj.startswith(".") or proj.startswith("@"):
             continue
+        # opt-out: a project marked .apex-root-owned runs as root and keeps its tier
+        # data root-owned (e.g. seafile-mc, whose entrypoint refuses non-root-owned
+        # data). Skip the noroot user + .env UID injection; tiers/chown forces root.
+        if os.path.isfile(os.path.join(full, ".apex-root-owned")):
+            log.info(f"{proj}: .apex-root-owned — runs as root; skipping noroot user")
+            continue
         base = f"noroot-{_sanitize(proj)}"
         cand, i = base, 1
         while cand in used:

assets/alloy/config.alloy

@@ -0,0 +1,120 @@
+// ============================================================================
+// APEX fleet observability agent — per-node Grafana Alloy config (shared).
+// Committed, env-parameterized: every node runs this SAME file; per-node values
+// come from sys.env(...) (APEX_NODE, push URLs, obs basic-auth creds).
+// Verified against grafana.com/docs/alloy/latest (Alloy v1.17.1).
+// ============================================================================
+// Model: PUSH. Metrics -> remote_write to the central Prometheus (on icarus);
+// logs -> loki.write to the central Loki (on icarus). daedalus/morpheus reach
+// them through the public gfn.ermnvldmr.com front (authelia-bypassed + basicauth
+// on /loki and /prometheus/api/v1/write); icarus resolves gfn locally.
+// ============================================================================
+
+// ----------------------------------------------------------------------------
+// METRICS
+// ----------------------------------------------------------------------------
+
+// Host metrics (node_exporter-equivalent). Container mounts host /proc,/sys,/
+// at /host/proc,/host/sys,/rootfs so the exporter reads the HOST.
+prometheus.exporter.unix "node" {
+  procfs_path = "/host/proc"
+  sysfs_path  = "/host/sys"
+  rootfs_path = "/rootfs"
+}
+
+// Container metrics (cadvisor-equivalent) via the docker socket.
+prometheus.exporter.cadvisor "containers" {
+  docker_host            = "unix:///var/run/docker.sock"
+  store_container_labels = true
+}
+
+// Stamp a static `node` label onto each exporter's targets.
+discovery.relabel "node_metrics" {
+  targets = prometheus.exporter.unix.node.targets
+  rule {
+    action       = "replace"
+    target_label = "node"
+    replacement  = sys.env("APEX_NODE")
+  }
+}
+discovery.relabel "container_metrics" {
+  targets = prometheus.exporter.cadvisor.containers.targets
+  rule {
+    action       = "replace"
+    target_label = "node"
+    replacement  = sys.env("APEX_NODE")
+  }
+}
+
+prometheus.scrape "node" {
+  targets         = discovery.relabel.node_metrics.output
+  forward_to      = [prometheus.remote_write.default.receiver]
+  scrape_interval = "15s"
+}
+prometheus.scrape "containers" {
+  targets         = discovery.relabel.container_metrics.output
+  forward_to      = [prometheus.remote_write.default.receiver]
+  scrape_interval = "15s"
+}
+
+// Push metrics to the central Prometheus (needs --web.enable-remote-write-receiver).
+prometheus.remote_write "default" {
+  endpoint {
+    url = sys.env("APEX_PROM_PUSH_URL")
+    basic_auth {
+      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
+      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
+    }
+  }
+}
+
+// ----------------------------------------------------------------------------
+// LOGS
+// ----------------------------------------------------------------------------
+
+// systemd journal (host services, ufw, sshd, crowdsec, etc.).
+loki.source.journal "systemd" {
+  path    = "/var/log/journal"
+  max_age = "12h"
+  labels  = {
+    node = sys.env("APEX_NODE"),
+    job  = "systemd-journal",
+  }
+  forward_to = [loki.write.default.receiver]
+}
+
+// All docker containers' stdout/stderr.
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+discovery.relabel "container_logs" {
+  targets = discovery.docker.containers.targets
+  rule {
+    action        = "replace"
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/?(.*)"
+    target_label  = "container"
+  }
+  rule {
+    action       = "replace"
+    target_label = "node"
+    replacement  = sys.env("APEX_NODE")
+  }
+}
+loki.source.docker "containers" {
+  host          = "unix:///var/run/docker.sock"
+  targets       = discovery.relabel.container_logs.output
+  relabel_rules = discovery.relabel.container_logs.rules
+  forward_to    = [loki.write.default.receiver]
+}
+
+// Push logs to the central Loki.
+loki.write "default" {
+  endpoint {
+    url = sys.env("APEX_LOKI_PUSH_URL")
+    basic_auth {
+      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
+      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
+    }
+  }
+}

compositions/apex/docker-compose.yml

@@ -152,27 +152,48 @@ services:
     security_opt:
       - no-new-privileges:true
 
-  # --- optional: loki log aggregation (node-local, enclave) ---
-  loki:
-    image: grafana/loki:3.1.1
-    container_name: apex-loki
-    profiles: [loki]
-    command: ["-config.file=/etc/loki/config.yaml"]
+  # --- optional: observability agent (profile-gated) ---
+  # Grafana Alloy: one agent per node. Collects host metrics (node_exporter-
+  # equivalent) + container metrics (cadvisor) and PUSHES them via remote_write;
+  # collects systemd-journal + docker logs and PUSHES them to Loki. All targets
+  # are OUTBOUND to the central gfn stack (Prometheus + Loki live on icarus), so
+  # Alloy needs egress (direct, not enclave). Replaces the old store-only `loki`
+  # service (which had no feeder). Config is a committed, env-parameterized asset.
+  alloy:
+    image: grafana/alloy:v1.17.1
+    container_name: apex-alloy
+    profiles: [alloy]
+    user: "0:0"  # root: required for host metrics, journal, and docker.sock
     volumes:
-      - ${APEX_TIER1}/loki/config.yaml:/etc/loki/config.yaml:ro
-      - ${APEX_TIER2}/loki/data:/loki
+      - ${APEX_COMMONS}/assets/alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - ${APEX_TIER2}/alloy/data:/var/lib/alloy
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /var/log/journal:/var/log/journal:ro
+      - /run/log/journal:/run/log/journal:ro
+      - /etc/machine-id:/etc/machine-id:ro
+    environment:
+      <<: *shared-environment
+      APEX_NODE: "${APEX_NODE?error}"
+      # push targets + machine auth (per-node scalars in apex.env / secret .env)
+      APEX_LOKI_PUSH_URL: "${APEX_LOKI_PUSH_URL?error}"
+      APEX_PROM_PUSH_URL: "${APEX_PROM_PUSH_URL?error}"
+      APEX_OBS_BASIC_AUTH_USER: "${APEX_OBS_BASIC_AUTH_USER?error}"
+      APEX_OBS_BASIC_AUTH_PASS: "${APEX_OBS_BASIC_AUTH_PASS?error}"
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy
+      - /etc/alloy/config.alloy
     networks:
-      enclave: {}
+      direct: {}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     restart: unless-stopped
-    user: "${APEX_UID?error}:${APEX_GID?error}"
-    labels:
-      # /loki/* is loki's native API prefix — no strip needed. basicauth users come
-      # from the node's uncommitted .env (only needed when the loki profile is on).
-      - "traefik.enable=true"
-      - "traefik.http.routers.apex-loki.rule=Host(`${APEX_NODE_FQDN}`) && PathPrefix(`/loki`)"
-      - "traefik.http.routers.apex-loki.middlewares=apex-loki-auth"
-      - "traefik.http.middlewares.apex-loki-auth.basicauth.users=${APEX_LOKI_BASIC_AUTH}"
-      - "traefik.http.services.apex-loki.loadbalancer.server.port=3100"
+    security_opt:
+      - no-new-privileges:true
 
 networks:
   direct: