deytenit
bd973d783d
Drop the hardcoded-domain FQDN regex and every hardcoded domain from commons. Node identity is now explicit in node.env: APEX_NODE_FQDN (real FQDN, traefik Host()), APEX_NODE_HOST (public host name — backup --host, obs/notify instance), APEX_SUBNET. The short name (log prefix, sync/<node> branch, messages) is derived as the first host label. notify instance is just the host (no hardcoded prefix); the git bot identity moves to APEX_GIT_AUTHOR_NAME/EMAIL (node.env) with generic defaults. Alloy node+instance labels use APEX_NODE_HOST. Fallbacks keep a node working if it omits a var (host<-fqdn<-hostname). BREAKING CHANGE: removes APEX_NODE and APEX_CLUSTER env vars; node name is now derived from APEX_NODE_FQDN
132 lines
4.2 KiB
Plaintext
132 lines
4.2 KiB
Plaintext
// ============================================================================
|
|
// APEX fleet observability agent — per-node Grafana Alloy config (shared).
|
|
// Committed, env-parameterized: every node runs this SAME file; per-node values
|
|
// come from sys.env(...) (APEX_NODE_HOST, push URLs, obs basic-auth creds).
|
|
// Verified against grafana.com/docs/alloy/latest (Alloy v1.17.1).
|
|
// ============================================================================
|
|
// Model: PUSH. Metrics -> remote_write to a central Prometheus; logs -> loki.write
|
|
// to a central Loki. Push endpoints + basic-auth are per-node env
|
|
// (APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, APEX_OBS_BASIC_AUTH_USER/PASS).
|
|
// ============================================================================
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// METRICS
|
|
// ----------------------------------------------------------------------------
|
|
|
|
// Host metrics (node_exporter-equivalent). Container mounts host /proc,/sys,/
|
|
// at /host/proc,/host/sys,/rootfs so the exporter reads the HOST.
|
|
prometheus.exporter.unix "node" {
|
|
procfs_path = "/host/proc"
|
|
sysfs_path = "/host/sys"
|
|
rootfs_path = "/rootfs"
|
|
}
|
|
|
|
// Container metrics (cadvisor-equivalent) via the docker socket.
|
|
prometheus.exporter.cadvisor "containers" {
|
|
docker_host = "unix:///var/run/docker.sock"
|
|
store_container_labels = true
|
|
}
|
|
|
|
// Stamp a static `node` label onto each exporter's targets.
|
|
discovery.relabel "node_metrics" {
|
|
targets = prometheus.exporter.unix.node.targets
|
|
rule {
|
|
action = "replace"
|
|
target_label = "node"
|
|
replacement = sys.env("APEX_NODE_HOST")
|
|
}
|
|
// Stamp a stable `instance` (the node's public host) instead of the default,
|
|
// which is the Alloy container id/address.
|
|
rule {
|
|
action = "replace"
|
|
target_label = "instance"
|
|
replacement = sys.env("APEX_NODE_HOST")
|
|
}
|
|
}
|
|
discovery.relabel "container_metrics" {
|
|
targets = prometheus.exporter.cadvisor.containers.targets
|
|
rule {
|
|
action = "replace"
|
|
target_label = "node"
|
|
replacement = sys.env("APEX_NODE_HOST")
|
|
}
|
|
rule {
|
|
action = "replace"
|
|
target_label = "instance"
|
|
replacement = sys.env("APEX_NODE_HOST")
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "node" {
|
|
targets = discovery.relabel.node_metrics.output
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
scrape_interval = "15s"
|
|
}
|
|
prometheus.scrape "containers" {
|
|
targets = discovery.relabel.container_metrics.output
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
scrape_interval = "15s"
|
|
}
|
|
|
|
// Push metrics to the central Prometheus (needs --web.enable-remote-write-receiver).
|
|
prometheus.remote_write "default" {
|
|
endpoint {
|
|
url = sys.env("APEX_PROM_PUSH_URL")
|
|
basic_auth {
|
|
username = sys.env("APEX_OBS_BASIC_AUTH_USER")
|
|
password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
|
|
}
|
|
}
|
|
}
|
|
|
|
// ----------------------------------------------------------------------------
|
|
// LOGS
|
|
// ----------------------------------------------------------------------------
|
|
|
|
// systemd journal (host services, ufw, sshd, crowdsec, etc.).
|
|
loki.source.journal "systemd" {
|
|
path = "/var/log/journal"
|
|
max_age = "12h"
|
|
labels = {
|
|
node = sys.env("APEX_NODE_HOST"),
|
|
job = "systemd-journal",
|
|
}
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
// All docker containers' stdout/stderr.
|
|
discovery.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
}
|
|
discovery.relabel "container_logs" {
|
|
targets = discovery.docker.containers.targets
|
|
rule {
|
|
action = "replace"
|
|
source_labels = ["__meta_docker_container_name"]
|
|
regex = "/?(.*)"
|
|
target_label = "container"
|
|
}
|
|
rule {
|
|
action = "replace"
|
|
target_label = "node"
|
|
replacement = sys.env("APEX_NODE_HOST")
|
|
}
|
|
}
|
|
loki.source.docker "containers" {
|
|
host = "unix:///var/run/docker.sock"
|
|
targets = discovery.relabel.container_logs.output
|
|
relabel_rules = discovery.relabel.container_logs.rules
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
// Push logs to the central Loki.
|
|
loki.write "default" {
|
|
endpoint {
|
|
url = sys.env("APEX_LOKI_PUSH_URL")
|
|
basic_auth {
|
|
username = sys.env("APEX_OBS_BASIC_AUTH_USER")
|
|
password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
|
|
}
|
|
}
|
|
}
|