diff --git a/.gitignore b/.gitignore index 8eb9168..7b6350c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ __pycache__/ *.env .env *.bak +._* +.DS_Store diff --git a/actions/configure/alloy.py b/actions/configure/alloy.py new file mode 100644 index 0000000..f01ac3a --- /dev/null +++ b/actions/configure/alloy.py @@ -0,0 +1,77 @@ +"""Deploy the opt-in native Alloy service using shared host/guest configs.""" +import json +from pathlib import Path +import re +import tempfile + +from engine.descriptor import Flag, Meta + +METADATA = Meta(summary="Configure native Alloy from configs/alloy/config.json.", + args=[Flag("--dry-run", "Validate the deployment plan without changes.")]) + + +def deployment(ctx): + settings = Path(ctx.paths.configs) / "alloy/config.json" + if not settings.exists(): + return None + values = json.loads(settings.read_text()) + role = values.get("role") + if role not in ("host", "guest"): + raise ValueError("Alloy role must be host or guest") + env_file = values.get("environment_file", "") + if not re.fullmatch(r"/[A-Za-z0-9_./-]+", env_file): + raise ValueError("Alloy environment_file must be an absolute path without whitespace") + source = Path(ctx.paths.commons) / "etc/alloy" / ( + "native-host.alloy" if role == "host" else "native.alloy") + config = source.read_text() + extra = Path(ctx.paths.configs) / "alloy/extra.alloy" + if extra.is_file(): + config += "\n" + extra.read_text() + unit = ("[Service]\nEnvironmentFile=" + env_file + "\n" + 'ExecStart=\nExecStart=/usr/bin/alloy run --storage.path=/var/lib/alloy --server.http.listen-addr=127.0.0.1:12346 /etc/alloy/config.alloy\n' + + ("User=root\nGroup=root\n" if role == "guest" else "") + + "SupplementaryGroups=systemd-journal" + (" docker" if role == "guest" else "") + "\n") + return config, unit, env_file + + +def run(ctx, args): + plan = deployment(ctx) + if plan is None: + ctx.log.info("Native Alloy is not configured for this node; skipping.") + return + config, unit, env_file = plan + if args.dry_run: + ctx.log.info("Native Alloy deployment plan validated; no changes made.") + return + ctx.sys.sudo(["test", "-f", env_file]) + # Validate with the actual service environment without printing credentials. + validator = '''import os, stat, subprocess, sys +from pathlib import Path +# Environment files deliberately use plain KEY=value, with no shell evaluation. +secret = Path(sys.argv[1]) +info = secret.stat() +if info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o077: + raise RuntimeError('Alloy environment file must be root-owned with mode 0600 or stricter') +for line in secret.read_text().splitlines(): + if not line.strip() or line.lstrip().startswith('#'): continue + key, value = line.split('=', 1) + os.environ[key] = value +subprocess.run(['alloy', 'validate', sys.argv[2]], check=True) +''' + with tempfile.TemporaryDirectory(prefix="apex-alloy-") as directory: + config_path = Path(directory) / "config.alloy" + unit_path = Path(directory) / "apex.conf" + config_path.write_text(config) + unit_path.write_text(unit) + ctx.sys.sudo(["python3", "-c", validator, env_file, str(config_path)]) + ctx.sys.sudo(["install", "-d", "-m", "0755", "/etc/alloy", + "/etc/systemd/system/alloy.service.d"]) + ctx.sys.sudo(["install", "-m", "0644", str(config_path), "/etc/alloy/config.alloy"]) + ctx.sys.sudo(["install", "-m", "0644", str(unit_path), + "/etc/systemd/system/alloy.service.d/apex.conf"]) + ctx.sys.sudo(["systemctl", "unmask", "alloy.service"]) + ctx.sys.daemon_reload() + ctx.sys.sudo(["systemctl", "enable", "alloy.service"]) + ctx.sys.sudo(["systemctl", "restart", "alloy.service"]) + ctx.sys.sudo(["systemctl", "is-active", "--quiet", "alloy.service"]) + ctx.log.success("Configured native Alloy.") diff --git a/etc/alloy/native-host.alloy b/etc/alloy/native-host.alloy new file mode 100644 index 0000000..88259c0 --- /dev/null +++ b/etc/alloy/native-host.alloy @@ -0,0 +1,102 @@ +// Shared native Grafana Alloy configuration for the physical host. +// Metrics and logs use the endpoints and credentials supplied through +// APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, and APEX_OBS_BASIC_AUTH_USER/PASS. + +// ---------------------------------------------------------------------------- +// METRICS +// ---------------------------------------------------------------------------- + +// Native filesystem views; physical capacity is collected on the Proxmox host. +prometheus.exporter.unix "node" { + enable_collectors = ["textfile"] + textfile { + directory = "/var/lib/alloy/textfile" + } + procfs_path = "/proc" + sysfs_path = "/sys" + rootfs_path = "/" +} + +// Container metrics (cadvisor-equivalent) via the docker socket. + + +// Stamp a static `node` label onto each exporter's targets. +discovery.relabel "node_metrics" { + targets = prometheus.exporter.unix.node.targets + rule { + action = "replace" + target_label = "node" + replacement = sys.env("APEX_NODE_HOST") + } + // Stamp a stable `instance` (the node's public host) instead of the default, + // which is the Alloy container id/address. + rule { + action = "replace" + target_label = "instance" + replacement = sys.env("APEX_NODE_HOST") + } +} + + +prometheus.scrape "node" { + targets = discovery.relabel.node_metrics.output + forward_to = [prometheus.remote_write.default.receiver] + scrape_interval = "30s" +} + + +// The configured Prometheus-compatible endpoint must accept remote writes. +prometheus.remote_write "default" { + endpoint { + url = sys.env("APEX_PROM_PUSH_URL") + basic_auth { + username = sys.env("APEX_OBS_BASIC_AUTH_USER") + password = sys.env("APEX_OBS_BASIC_AUTH_PASS") + } + queue_config { + capacity = 5000 + max_shards = 10 + min_shards = 1 + max_samples_per_send = 1000 + batch_send_deadline = "5s" + min_backoff = "100ms" + max_backoff = "10s" + } + } + wal { + truncate_frequency = "30m" + min_keepalive_time = "5m" + max_keepalive_time = "30m" + } +} + +// ---------------------------------------------------------------------------- +// LOGS +// ---------------------------------------------------------------------------- + +// systemd journal (host services, ufw, sshd, crowdsec, etc.). +loki.source.journal "systemd" { + path = "/var/log/journal" + max_age = "12h" + labels = { + node = sys.env("APEX_NODE_HOST"), + job = "systemd-journal", + } + forward_to = [loki.write.default.receiver] +} + +// All docker containers' stdout/stderr. + + + + +// Push logs to the central Loki. +loki.write "default" { + endpoint { + url = sys.env("APEX_LOKI_PUSH_URL") + basic_auth { + username = sys.env("APEX_OBS_BASIC_AUTH_USER") + password = sys.env("APEX_OBS_BASIC_AUTH_PASS") + } + } +} diff --git a/etc/alloy/native.alloy b/etc/alloy/native.alloy new file mode 100644 index 0000000..2a157d3 --- /dev/null +++ b/etc/alloy/native.alloy @@ -0,0 +1,128 @@ +// Shared native Grafana Alloy configuration for Docker guests. +// Metrics and logs use the endpoints and credentials supplied through +// APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, and APEX_OBS_BASIC_AUTH_USER/PASS. + +// ---------------------------------------------------------------------------- +// METRICS +// ---------------------------------------------------------------------------- + +// Native filesystem views; physical capacity is collected on the Proxmox host. + + +// Container metrics (cadvisor-equivalent) via the docker socket. +prometheus.exporter.cadvisor "containers" { + docker_host = "unix:///var/run/docker.sock" + store_container_labels = false + allowlisted_container_labels = [ + "com.docker.compose.project", + "com.docker.compose.service", + ] + docker_only = true + enabled_metrics = [ + "cpu", + "cpuLoad", + "diskIO", + "memory", + "network", + "oom_event", + ] +} + +// Stamp a static `node` label onto each exporter's targets. + +discovery.relabel "container_metrics" { + targets = prometheus.exporter.cadvisor.containers.targets + rule { + action = "replace" + target_label = "node" + replacement = sys.env("APEX_NODE_HOST") + } + rule { + action = "replace" + target_label = "instance" + replacement = sys.env("APEX_NODE_HOST") + } +} + + +prometheus.scrape "containers" { + targets = discovery.relabel.container_metrics.output + forward_to = [prometheus.remote_write.default.receiver] + scrape_interval = "30s" +} + +// The configured Prometheus-compatible endpoint must accept remote writes. +prometheus.remote_write "default" { + endpoint { + url = sys.env("APEX_PROM_PUSH_URL") + basic_auth { + username = sys.env("APEX_OBS_BASIC_AUTH_USER") + password = sys.env("APEX_OBS_BASIC_AUTH_PASS") + } + queue_config { + capacity = 5000 + max_shards = 10 + min_shards = 1 + max_samples_per_send = 1000 + batch_send_deadline = "5s" + min_backoff = "100ms" + max_backoff = "10s" + } + } + wal { + truncate_frequency = "30m" + min_keepalive_time = "5m" + max_keepalive_time = "30m" + } +} + +// ---------------------------------------------------------------------------- +// LOGS +// ---------------------------------------------------------------------------- + +// systemd journal (host services, ufw, sshd, crowdsec, etc.). +loki.source.journal "systemd" { + path = "/var/log/journal" + max_age = "12h" + labels = { + node = sys.env("APEX_NODE_HOST"), + job = "systemd-journal", + } + forward_to = [loki.write.default.receiver] +} + +// All docker containers' stdout/stderr. +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" +} +discovery.relabel "container_logs" { + targets = discovery.docker.containers.targets + rule { + action = "replace" + source_labels = ["__meta_docker_container_name"] + regex = "/?(.*)" + target_label = "container" + } + rule { + action = "replace" + target_label = "node" + replacement = sys.env("APEX_NODE_HOST") + } +} +loki.source.docker "containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.container_logs.output + relabel_rules = discovery.relabel.container_logs.rules + forward_to = [loki.write.default.receiver] +} + +// Push logs to the central Loki. +loki.write "default" { + endpoint { + url = sys.env("APEX_LOKI_PUSH_URL") + basic_auth { + username = sys.env("APEX_OBS_BASIC_AUTH_USER") + password = sys.env("APEX_OBS_BASIC_AUTH_PASS") + } + } +}