From b41fc6097f29f3bd5c4a6daeac2e2e58d4e40721 Mon Sep 17 00:00:00 2001 From: Vladimir Eremin Date: Mon, 6 Jul 2026 14:29:21 +0300 Subject: [PATCH] refactor: replace loki store with Grafana Alloy push agent --- assets/alloy/config.alloy | 120 +++++++++++++++++++++++++++ compositions/apex/docker-compose.yml | 57 +++++++++---- 2 files changed, 159 insertions(+), 18 deletions(-) create mode 100644 assets/alloy/config.alloy diff --git a/assets/alloy/config.alloy b/assets/alloy/config.alloy new file mode 100644 index 0000000..367f32e --- /dev/null +++ b/assets/alloy/config.alloy @@ -0,0 +1,120 @@ +// ============================================================================ +// APEX fleet observability agent — per-node Grafana Alloy config (shared). +// Committed, env-parameterized: every node runs this SAME file; per-node values +// come from sys.env(...) (APEX_NODE, push URLs, obs basic-auth creds). +// Verified against grafana.com/docs/alloy/latest (Alloy v1.17.1). +// ============================================================================ +// Model: PUSH. Metrics -> remote_write to the central Prometheus (on icarus); +// logs -> loki.write to the central Loki (on icarus). daedalus/morpheus reach +// them through the public gfn.ermnvldmr.com front (authelia-bypassed + basicauth +// on /loki and /prometheus/api/v1/write); icarus resolves gfn locally. +// ============================================================================ + +// ---------------------------------------------------------------------------- +// METRICS +// ---------------------------------------------------------------------------- + +// Host metrics (node_exporter-equivalent). Container mounts host /proc,/sys,/ +// at /host/proc,/host/sys,/rootfs so the exporter reads the HOST. +prometheus.exporter.unix "node" { + procfs_path = "/host/proc" + sysfs_path = "/host/sys" + rootfs_path = "/rootfs" +} + +// Container metrics (cadvisor-equivalent) via the docker socket. +prometheus.exporter.cadvisor "containers" { + docker_host = "unix:///var/run/docker.sock" + store_container_labels = true +} + +// Stamp a static `node` label onto each exporter's targets. +discovery.relabel "node_metrics" { + targets = prometheus.exporter.unix.node.targets + rule { + action = "replace" + target_label = "node" + replacement = sys.env("APEX_NODE") + } +} +discovery.relabel "container_metrics" { + targets = prometheus.exporter.cadvisor.containers.targets + rule { + action = "replace" + target_label = "node" + replacement = sys.env("APEX_NODE") + } +} + +prometheus.scrape "node" { + targets = discovery.relabel.node_metrics.output + forward_to = [prometheus.remote_write.default.receiver] + scrape_interval = "15s" +} +prometheus.scrape "containers" { + targets = discovery.relabel.container_metrics.output + forward_to = [prometheus.remote_write.default.receiver] + scrape_interval = "15s" +} + +// Push metrics to the central Prometheus (needs --web.enable-remote-write-receiver). +prometheus.remote_write "default" { + endpoint { + url = sys.env("APEX_PROM_PUSH_URL") + basic_auth { + username = sys.env("APEX_OBS_BASIC_AUTH_USER") + password = sys.env("APEX_OBS_BASIC_AUTH_PASS") + } + } +} + +// ---------------------------------------------------------------------------- +// LOGS +// ---------------------------------------------------------------------------- + +// systemd journal (host services, ufw, sshd, crowdsec, etc.). +loki.source.journal "systemd" { + path = "/var/log/journal" + max_age = "12h" + labels = { + node = sys.env("APEX_NODE"), + job = "systemd-journal", + } + forward_to = [loki.write.default.receiver] +} + +// All docker containers' stdout/stderr. +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" +} +discovery.relabel "container_logs" { + targets = discovery.docker.containers.targets + rule { + action = "replace" + source_labels = ["__meta_docker_container_name"] + regex = "/?(.*)" + target_label = "container" + } + rule { + action = "replace" + target_label = "node" + replacement = sys.env("APEX_NODE") + } +} +loki.source.docker "containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.container_logs.output + relabel_rules = discovery.relabel.container_logs.rules + forward_to = [loki.write.default.receiver] +} + +// Push logs to the central Loki. +loki.write "default" { + endpoint { + url = sys.env("APEX_LOKI_PUSH_URL") + basic_auth { + username = sys.env("APEX_OBS_BASIC_AUTH_USER") + password = sys.env("APEX_OBS_BASIC_AUTH_PASS") + } + } +} diff --git a/compositions/apex/docker-compose.yml b/compositions/apex/docker-compose.yml index 94ede5c..2bd96a2 100644 --- a/compositions/apex/docker-compose.yml +++ b/compositions/apex/docker-compose.yml @@ -152,27 +152,48 @@ services: security_opt: - no-new-privileges:true - # --- optional: loki log aggregation (node-local, enclave) --- - loki: - image: grafana/loki:3.1.1 - container_name: apex-loki - profiles: [loki] - command: ["-config.file=/etc/loki/config.yaml"] + # --- optional: observability agent (profile-gated) --- + # Grafana Alloy: one agent per node. Collects host metrics (node_exporter- + # equivalent) + container metrics (cadvisor) and PUSHES them via remote_write; + # collects systemd-journal + docker logs and PUSHES them to Loki. All targets + # are OUTBOUND to the central gfn stack (Prometheus + Loki live on icarus), so + # Alloy needs egress (direct, not enclave). Replaces the old store-only `loki` + # service (which had no feeder). Config is a committed, env-parameterized asset. + alloy: + image: grafana/alloy:v1.17.1 + container_name: apex-alloy + profiles: [alloy] + user: "0:0" # root: required for host metrics, journal, and docker.sock volumes: - - ${APEX_TIER1}/loki/config.yaml:/etc/loki/config.yaml:ro - - ${APEX_TIER2}/loki/data:/loki + - ${APEX_COMMONS}/assets/alloy/config.alloy:/etc/alloy/config.alloy:ro + - ${APEX_TIER2}/alloy/data:/var/lib/alloy + - /var/run/docker.sock:/var/run/docker.sock:ro + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + - /var/log/journal:/var/log/journal:ro + - /run/log/journal:/run/log/journal:ro + - /etc/machine-id:/etc/machine-id:ro + environment: + <<: *shared-environment + APEX_NODE: "${APEX_NODE?error}" + # push targets + machine auth (per-node scalars in apex.env / secret .env) + APEX_LOKI_PUSH_URL: "${APEX_LOKI_PUSH_URL?error}" + APEX_PROM_PUSH_URL: "${APEX_PROM_PUSH_URL?error}" + APEX_OBS_BASIC_AUTH_USER: "${APEX_OBS_BASIC_AUTH_USER?error}" + APEX_OBS_BASIC_AUTH_PASS: "${APEX_OBS_BASIC_AUTH_PASS?error}" + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy + - /etc/alloy/config.alloy networks: - enclave: {} + direct: {} + extra_hosts: + - "host.docker.internal:host-gateway" restart: unless-stopped - user: "${APEX_UID?error}:${APEX_GID?error}" - labels: - # /loki/* is loki's native API prefix — no strip needed. basicauth users come - # from the node's uncommitted .env (only needed when the loki profile is on). - - "traefik.enable=true" - - "traefik.http.routers.apex-loki.rule=Host(`${APEX_NODE_FQDN}`) && PathPrefix(`/loki`)" - - "traefik.http.routers.apex-loki.middlewares=apex-loki-auth" - - "traefik.http.middlewares.apex-loki-auth.basicauth.users=${APEX_LOKI_BASIC_AUTH}" - - "traefik.http.services.apex-loki.loadbalancer.server.port=3100" + security_opt: + - no-new-privileges:true networks: direct: