feat(engine): de-slop commons, harden security, add targeted compose and env bridge
- Purge misplaced RFCs, scratch tokens, and node-proprietary assets - Decouple remote provisioning to ci/scripts/init-node.py - Add targeted composition orchestration in actions/compose.py - Add host-side Xray stale socket cleanup in /dev/shm - Add actions/env.py for shell exports and .env synchronization - Harden UFW, backup retention, host tempfiles, and notify dispatcher - Add pyproject.toml and Makefile with 78 passing unit tests
This commit is contained in:
66 files changed
+686
-1657
No files matched your search
@@ -0,0 +1,173 @@
|
||||
// ============================================================================
|
||||
// APEX fleet observability agent — per-node Grafana Alloy config (shared).
|
||||
// Committed, env-parameterized: every node runs this SAME file; per-node values
|
||||
// come from sys.env(...) (APEX_NODE_HOST, push URLs, obs basic-auth creds).
|
||||
// Verified against grafana.com/docs/alloy/latest (Alloy v1.17.1).
|
||||
// ============================================================================
|
||||
// Model: PUSH. Metrics -> remote_write to a central Prometheus; logs -> loki.write
|
||||
// to a central Loki. Push endpoints + basic-auth are per-node env
|
||||
// (APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, APEX_OBS_BASIC_AUTH_USER/PASS).
|
||||
// ============================================================================
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// METRICS
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
// Host metrics (node_exporter-equivalent). Container mounts host /proc,/sys,/
|
||||
// at /host/proc,/host/sys,/rootfs so the exporter reads the HOST.
|
||||
prometheus.exporter.unix "node" {
|
||||
procfs_path = "/host/proc"
|
||||
sysfs_path = "/host/sys"
|
||||
rootfs_path = "/rootfs"
|
||||
}
|
||||
|
||||
// Container metrics (cadvisor-equivalent) via the docker socket.
|
||||
prometheus.exporter.cadvisor "containers" {
|
||||
docker_host = "unix:///var/run/docker.sock"
|
||||
store_container_labels = false
|
||||
allowlisted_container_labels = [
|
||||
"com.docker.compose.project",
|
||||
"com.docker.compose.service",
|
||||
]
|
||||
docker_only = true
|
||||
enabled_metrics = [
|
||||
"cpu",
|
||||
"cpuLoad",
|
||||
"diskIO",
|
||||
"memory",
|
||||
"network",
|
||||
"oom_event",
|
||||
]
|
||||
}
|
||||
|
||||
// Stamp a static `node` label onto each exporter's targets.
|
||||
discovery.relabel "node_metrics" {
|
||||
targets = prometheus.exporter.unix.node.targets
|
||||
rule {
|
||||
action = "replace"
|
||||
target_label = "node"
|
||||
replacement = sys.env("APEX_NODE_HOST")
|
||||
}
|
||||
// Stamp a stable `instance` (the node's public host) instead of the default,
|
||||
// which is the Alloy container id/address.
|
||||
rule {
|
||||
action = "replace"
|
||||
target_label = "instance"
|
||||
replacement = sys.env("APEX_NODE_HOST")
|
||||
}
|
||||
}
|
||||
discovery.relabel "container_metrics" {
|
||||
targets = prometheus.exporter.cadvisor.containers.targets
|
||||
rule {
|
||||
action = "replace"
|
||||
target_label = "node"
|
||||
replacement = sys.env("APEX_NODE_HOST")
|
||||
}
|
||||
rule {
|
||||
action = "replace"
|
||||
target_label = "instance"
|
||||
replacement = sys.env("APEX_NODE_HOST")
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "node" {
|
||||
targets = discovery.relabel.node_metrics.output
|
||||
forward_to = [prometheus.remote_write.default.receiver]
|
||||
scrape_interval = "30s"
|
||||
}
|
||||
prometheus.scrape "containers" {
|
||||
targets = discovery.relabel.container_metrics.output
|
||||
forward_to = [prometheus.remote_write.default.receiver]
|
||||
scrape_interval = "30s"
|
||||
}
|
||||
|
||||
// CrowdSec agent metrics. CrowdSec runs on the host (not a container) and exposes Prometheus
|
||||
// metrics on :6060; nodes that run it bind that endpoint on 0.0.0.0 (the host firewall keeps it
|
||||
// off the public interface) so Alloy can reach it via the docker gateway. Nodes that don't run
|
||||
// CrowdSec simply report this target down.
|
||||
prometheus.scrape "crowdsec" {
|
||||
targets = [{
|
||||
__address__ = "host.docker.internal:6060",
|
||||
instance = sys.env("APEX_NODE_HOST"),
|
||||
node = sys.env("APEX_NODE_HOST"),
|
||||
job = "crowdsec",
|
||||
}]
|
||||
forward_to = [prometheus.remote_write.default.receiver]
|
||||
scrape_interval = "30s"
|
||||
}
|
||||
|
||||
// Push metrics to the central Prometheus (needs --web.enable-remote-write-receiver).
|
||||
prometheus.remote_write "default" {
|
||||
endpoint {
|
||||
url = sys.env("APEX_PROM_PUSH_URL")
|
||||
basic_auth {
|
||||
username = sys.env("APEX_OBS_BASIC_AUTH_USER")
|
||||
password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
|
||||
}
|
||||
queue_config {
|
||||
capacity = 5000
|
||||
max_shards = 10
|
||||
min_shards = 1
|
||||
max_samples_per_send = 1000
|
||||
batch_send_deadline = "5s"
|
||||
min_backoff = "100ms"
|
||||
max_backoff = "10s"
|
||||
}
|
||||
}
|
||||
wal {
|
||||
truncate_frequency = "30m"
|
||||
min_keepalive_time = "5m"
|
||||
max_keepalive_time = "30m"
|
||||
}
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// LOGS
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
// systemd journal (host services, ufw, sshd, crowdsec, etc.).
|
||||
loki.source.journal "systemd" {
|
||||
path = "/var/log/journal"
|
||||
max_age = "12h"
|
||||
labels = {
|
||||
node = sys.env("APEX_NODE_HOST"),
|
||||
job = "systemd-journal",
|
||||
}
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
// All docker containers' stdout/stderr.
|
||||
discovery.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
}
|
||||
discovery.relabel "container_logs" {
|
||||
targets = discovery.docker.containers.targets
|
||||
rule {
|
||||
action = "replace"
|
||||
source_labels = ["__meta_docker_container_name"]
|
||||
regex = "/?(.*)"
|
||||
target_label = "container"
|
||||
}
|
||||
rule {
|
||||
action = "replace"
|
||||
target_label = "node"
|
||||
replacement = sys.env("APEX_NODE_HOST")
|
||||
}
|
||||
}
|
||||
loki.source.docker "containers" {
|
||||
host = "unix:///var/run/docker.sock"
|
||||
targets = discovery.relabel.container_logs.output
|
||||
relabel_rules = discovery.relabel.container_logs.rules
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
// Push logs to the central Loki.
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = sys.env("APEX_LOKI_PUSH_URL")
|
||||
basic_auth {
|
||||
username = sys.env("APEX_OBS_BASIC_AUTH_USER")
|
||||
password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user