apex.ermnvldmr.com/assets/alloy/config.alloy

// ============================================================================
// APEX fleet observability agent — per-node Grafana Alloy config (shared).
// Committed, env-parameterized: every node runs this SAME file; per-node values
// come from sys.env(...) (APEX_NODE_HOST, push URLs, obs basic-auth creds).
// Verified against grafana.com/docs/alloy/latest (Alloy v1.17.1).
// ============================================================================
// Model: PUSH. Metrics -> remote_write to a central Prometheus; logs -> loki.write
// to a central Loki. Push endpoints + basic-auth are per-node env
// (APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, APEX_OBS_BASIC_AUTH_USER/PASS).
// ============================================================================

// ----------------------------------------------------------------------------
// METRICS
// ----------------------------------------------------------------------------

// Host metrics (node_exporter-equivalent). Container mounts host /proc,/sys,/
// at /host/proc,/host/sys,/rootfs so the exporter reads the HOST.
prometheus.exporter.unix "node" {
  procfs_path = "/host/proc"
  sysfs_path  = "/host/sys"
  rootfs_path = "/rootfs"
}

// Container metrics (cadvisor-equivalent) via the docker socket.
prometheus.exporter.cadvisor "containers" {
  docker_host            = "unix:///var/run/docker.sock"
  store_container_labels = true
}

// Stamp a static `node` label onto each exporter's targets.
discovery.relabel "node_metrics" {
  targets = prometheus.exporter.unix.node.targets
  rule {
    action       = "replace"
    target_label = "node"
    replacement  = sys.env("APEX_NODE_HOST")
  }
  // Stamp a stable `instance` (the node's public host) instead of the default,
  // which is the Alloy container id/address.
  rule {
    action       = "replace"
    target_label = "instance"
    replacement  = sys.env("APEX_NODE_HOST")
  }
}
discovery.relabel "container_metrics" {
  targets = prometheus.exporter.cadvisor.containers.targets
  rule {
    action       = "replace"
    target_label = "node"
    replacement  = sys.env("APEX_NODE_HOST")
  }
  rule {
    action       = "replace"
    target_label = "instance"
    replacement  = sys.env("APEX_NODE_HOST")
  }
}

prometheus.scrape "node" {
  targets         = discovery.relabel.node_metrics.output
  forward_to      = [prometheus.remote_write.default.receiver]
  scrape_interval = "15s"
}
prometheus.scrape "containers" {
  targets         = discovery.relabel.container_metrics.output
  forward_to      = [prometheus.remote_write.default.receiver]
  scrape_interval = "15s"
}

// CrowdSec agent metrics. CrowdSec runs on the host (not a container) and exposes Prometheus
// metrics on :6060; nodes that run it bind that endpoint on 0.0.0.0 (the host firewall keeps it
// off the public interface) so Alloy can reach it via the docker gateway. Nodes that don't run
// CrowdSec simply report this target down.
prometheus.scrape "crowdsec" {
  targets = [{
    __address__ = "host.docker.internal:6060",
    instance    = sys.env("APEX_NODE_HOST"),
    node        = sys.env("APEX_NODE_HOST"),
    job         = "crowdsec",
  }]
  forward_to      = [prometheus.remote_write.default.receiver]
  scrape_interval = "30s"
}

// Push metrics to the central Prometheus (needs --web.enable-remote-write-receiver).
prometheus.remote_write "default" {
  endpoint {
    url = sys.env("APEX_PROM_PUSH_URL")
    basic_auth {
      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
    }
  }
}

// ----------------------------------------------------------------------------
// LOGS
// ----------------------------------------------------------------------------

// systemd journal (host services, ufw, sshd, crowdsec, etc.).
loki.source.journal "systemd" {
  path    = "/var/log/journal"
  max_age = "12h"
  labels  = {
    node = sys.env("APEX_NODE_HOST"),
    job  = "systemd-journal",
  }
  forward_to = [loki.write.default.receiver]
}

// All docker containers' stdout/stderr.
discovery.docker "containers" {
  host = "unix:///var/run/docker.sock"
}
discovery.relabel "container_logs" {
  targets = discovery.docker.containers.targets
  rule {
    action        = "replace"
    source_labels = ["__meta_docker_container_name"]
    regex         = "/?(.*)"
    target_label  = "container"
  }
  rule {
    action       = "replace"
    target_label = "node"
    replacement  = sys.env("APEX_NODE_HOST")
  }
}
loki.source.docker "containers" {
  host          = "unix:///var/run/docker.sock"
  targets       = discovery.relabel.container_logs.output
  relabel_rules = discovery.relabel.container_logs.rules
  forward_to    = [loki.write.default.receiver]
}

// Push logs to the central Loki.
loki.write "default" {
  endpoint {
    url = sys.env("APEX_LOKI_PUSH_URL")
    basic_auth {
      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
    }
  }
}