apex.ermnvldmr.com/etc/alloy/config.alloy

// ============================================================================
// APEX fleet observability agent — per-node Grafana Alloy config (shared).
// Committed, env-parameterized: every node runs this SAME file; per-node values
// come from sys.env(...) (APEX_NODE_HOST, push URLs, obs basic-auth creds).
// Verified against grafana.com/docs/alloy/latest (Alloy v1.17.1).
// ============================================================================
// Model: PUSH. Metrics -> remote_write to a central Prometheus; logs -> loki.write
// to a central Loki. Push endpoints + basic-auth are per-node env
// (APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, APEX_OBS_BASIC_AUTH_USER/PASS).
// ============================================================================

// ----------------------------------------------------------------------------
// METRICS
// ----------------------------------------------------------------------------

// Host metrics (node_exporter-equivalent). Container mounts host /proc,/sys,/
// at /host/proc,/host/sys,/rootfs so the exporter reads the HOST.
prometheus.exporter.unix "node" {
  procfs_path = "/host/proc"
  sysfs_path  = "/host/sys"
  rootfs_path = "/rootfs"
}

// Container metrics (cadvisor-equivalent) via the docker socket.
prometheus.exporter.cadvisor "containers" {
  docker_host                  = "unix:///var/run/docker.sock"
  store_container_labels       = false
  allowlisted_container_labels = [
    "com.docker.compose.project",
    "com.docker.compose.service",
  ]
  docker_only                  = true
  enabled_metrics              = [
    "cpu",
    "cpuLoad",
    "diskIO",
    "memory",
    "network",
    "oom_event",
  ]
}

// Stamp a static `node` label onto each exporter's targets.
discovery.relabel "node_metrics" {
  targets = prometheus.exporter.unix.node.targets
  rule {
    action       = "replace"
    target_label = "node"
    replacement  = sys.env("APEX_NODE_HOST")
  }
  // Stamp a stable `instance` (the node's public host) instead of the default,
  // which is the Alloy container id/address.
  rule {
    action       = "replace"
    target_label = "instance"
    replacement  = sys.env("APEX_NODE_HOST")
  }
}
discovery.relabel "container_metrics" {
  targets = prometheus.exporter.cadvisor.containers.targets
  rule {
    action       = "replace"
    target_label = "node"
    replacement  = sys.env("APEX_NODE_HOST")
  }
  rule {
    action       = "replace"
    target_label = "instance"
    replacement  = sys.env("APEX_NODE_HOST")
  }
}

prometheus.scrape "node" {
  targets         = discovery.relabel.node_metrics.output
  forward_to      = [prometheus.remote_write.default.receiver]
  scrape_interval = "30s"
}
prometheus.scrape "containers" {
  targets         = discovery.relabel.container_metrics.output
  forward_to      = [prometheus.remote_write.default.receiver]
  scrape_interval = "30s"
}

// CrowdSec agent metrics. CrowdSec runs on the host (not a container) and exposes Prometheus
// metrics on :6060; nodes that run it bind that endpoint on 0.0.0.0 (the host firewall keeps it
// off the public interface) so Alloy can reach it via the docker gateway. Nodes that don't run
// CrowdSec simply report this target down.
prometheus.scrape "crowdsec" {
  targets = [{
    __address__ = "host.docker.internal:6060",
    instance    = sys.env("APEX_NODE_HOST"),
    node        = sys.env("APEX_NODE_HOST"),
    job         = "crowdsec",
  }]
  forward_to      = [prometheus.remote_write.default.receiver]
  scrape_interval = "30s"
}

// Push metrics to the central Prometheus (needs --web.enable-remote-write-receiver).
prometheus.remote_write "default" {
  endpoint {
    url = sys.env("APEX_PROM_PUSH_URL")
    basic_auth {
      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
    }
    queue_config {
      capacity             = 5000
      max_shards           = 10
      min_shards           = 1
      max_samples_per_send = 1000
      batch_send_deadline  = "5s"
      min_backoff          = "100ms"
      max_backoff          = "10s"
    }
  }
  wal {
    truncate_frequency = "30m"
    min_keepalive_time = "5m"
    max_keepalive_time = "30m"
  }
}

// ----------------------------------------------------------------------------
// LOGS
// ----------------------------------------------------------------------------

// systemd journal (host services, ufw, sshd, crowdsec, etc.).
loki.source.journal "systemd" {
  path    = "/var/log/journal"
  max_age = "12h"
  labels  = {
    node = sys.env("APEX_NODE_HOST"),
    job  = "systemd-journal",
  }
  forward_to = [loki.write.default.receiver]
}

// All docker containers' stdout/stderr.
discovery.docker "containers" {
  host = "unix:///var/run/docker.sock"
}
discovery.relabel "container_logs" {
  targets = discovery.docker.containers.targets
  rule {
    action        = "replace"
    source_labels = ["__meta_docker_container_name"]
    regex         = "/?(.*)"
    target_label  = "container"
  }
  rule {
    action       = "replace"
    target_label = "node"
    replacement  = sys.env("APEX_NODE_HOST")
  }
}
loki.source.docker "containers" {
  host          = "unix:///var/run/docker.sock"
  targets       = discovery.relabel.container_logs.output
  relabel_rules = discovery.relabel.container_logs.rules
  forward_to    = [loki.write.default.receiver]
}

// Push logs to the central Loki.
loki.write "default" {
  endpoint {
    url = sys.env("APEX_LOKI_PUSH_URL")
    basic_auth {
      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
    }
  }
}