Add native Alloy configuration for isolated Icarus deployment

.gitignore

@@ -15,3 +15,5 @@ __pycache__/
 *.env
 .env
 *.bak
+._*
+.DS_Store

actions/configure/alloy.py

@@ -0,0 +1,77 @@
+"""Deploy the opt-in native Alloy service using shared host/guest configs."""
+import json
+from pathlib import Path
+import re
+import tempfile
+
+from engine.descriptor import Flag, Meta
+
+METADATA = Meta(summary="Configure native Alloy from configs/alloy/config.json.",
+                args=[Flag("--dry-run", "Validate the deployment plan without changes.")])
+
+
+def deployment(ctx):
+    settings = Path(ctx.paths.configs) / "alloy/config.json"
+    if not settings.exists():
+        return None
+    values = json.loads(settings.read_text())
+    role = values.get("role")
+    if role not in ("host", "guest"):
+        raise ValueError("Alloy role must be host or guest")
+    env_file = values.get("environment_file", "")
+    if not re.fullmatch(r"/[A-Za-z0-9_./-]+", env_file):
+        raise ValueError("Alloy environment_file must be an absolute path without whitespace")
+    source = Path(ctx.paths.commons) / "etc/alloy" / (
+        "native-host.alloy" if role == "host" else "native.alloy")
+    config = source.read_text()
+    extra = Path(ctx.paths.configs) / "alloy/extra.alloy"
+    if extra.is_file():
+        config += "\n" + extra.read_text()
+    unit = ("[Service]\nEnvironmentFile=" + env_file + "\n"
+            'ExecStart=\nExecStart=/usr/bin/alloy run --storage.path=/var/lib/alloy --server.http.listen-addr=127.0.0.1:12346 /etc/alloy/config.alloy\n'
+            + ("User=root\nGroup=root\n" if role == "guest" else "") +
+            "SupplementaryGroups=systemd-journal" + (" docker" if role == "guest" else "") + "\n")
+    return config, unit, env_file
+
+
+def run(ctx, args):
+    plan = deployment(ctx)
+    if plan is None:
+        ctx.log.info("Native Alloy is not configured for this node; skipping.")
+        return
+    config, unit, env_file = plan
+    if args.dry_run:
+        ctx.log.info("Native Alloy deployment plan validated; no changes made.")
+        return
+    ctx.sys.sudo(["test", "-f", env_file])
+    # Validate with the actual service environment without printing credentials.
+    validator = '''import os, stat, subprocess, sys
+from pathlib import Path
+# Environment files deliberately use plain KEY=value, with no shell evaluation.
+secret = Path(sys.argv[1])
+info = secret.stat()
+if info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o077:
+    raise RuntimeError('Alloy environment file must be root-owned with mode 0600 or stricter')
+for line in secret.read_text().splitlines():
+    if not line.strip() or line.lstrip().startswith('#'): continue
+    key, value = line.split('=', 1)
+    os.environ[key] = value
+subprocess.run(['alloy', 'validate', sys.argv[2]], check=True)
+'''
+    with tempfile.TemporaryDirectory(prefix="apex-alloy-") as directory:
+        config_path = Path(directory) / "config.alloy"
+        unit_path = Path(directory) / "apex.conf"
+        config_path.write_text(config)
+        unit_path.write_text(unit)
+        ctx.sys.sudo(["python3", "-c", validator, env_file, str(config_path)])
+        ctx.sys.sudo(["install", "-d", "-m", "0755", "/etc/alloy",
+                      "/etc/systemd/system/alloy.service.d"])
+        ctx.sys.sudo(["install", "-m", "0644", str(config_path), "/etc/alloy/config.alloy"])
+        ctx.sys.sudo(["install", "-m", "0644", str(unit_path),
+                      "/etc/systemd/system/alloy.service.d/apex.conf"])
+    ctx.sys.sudo(["systemctl", "unmask", "alloy.service"])
+    ctx.sys.daemon_reload()
+    ctx.sys.sudo(["systemctl", "enable", "alloy.service"])
+    ctx.sys.sudo(["systemctl", "restart", "alloy.service"])
+    ctx.sys.sudo(["systemctl", "is-active", "--quiet", "alloy.service"])
+    ctx.log.success("Configured native Alloy.")

etc/alloy/native-host.alloy

@@ -0,0 +1,102 @@
+// Shared native Grafana Alloy configuration for the physical host.
+// Metrics and logs use the endpoints and credentials supplied through
+// APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, and APEX_OBS_BASIC_AUTH_USER/PASS.
+
+// ----------------------------------------------------------------------------
+// METRICS
+// ----------------------------------------------------------------------------
+
+// Native filesystem views; physical capacity is collected on the Proxmox host.
+prometheus.exporter.unix "node" {
+  enable_collectors = ["textfile"]
+  textfile {
+    directory = "/var/lib/alloy/textfile"
+  }
+  procfs_path = "/proc"
+  sysfs_path  = "/sys"
+  rootfs_path = "/"
+}
+
+// Container metrics (cadvisor-equivalent) via the docker socket.
+
+
+// Stamp a static `node` label onto each exporter's targets.
+discovery.relabel "node_metrics" {
+  targets = prometheus.exporter.unix.node.targets
+  rule {
+    action       = "replace"
+    target_label = "node"
+    replacement  = sys.env("APEX_NODE_HOST")
+  }
+  // Stamp a stable `instance` (the node's public host) instead of the default,
+  // which is the Alloy container id/address.
+  rule {
+    action       = "replace"
+    target_label = "instance"
+    replacement  = sys.env("APEX_NODE_HOST")
+  }
+}
+
+
+prometheus.scrape "node" {
+  targets         = discovery.relabel.node_metrics.output
+  forward_to      = [prometheus.remote_write.default.receiver]
+  scrape_interval = "30s"
+}
+
+
+// The configured Prometheus-compatible endpoint must accept remote writes.
+prometheus.remote_write "default" {
+  endpoint {
+    url = sys.env("APEX_PROM_PUSH_URL")
+    basic_auth {
+      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
+      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
+    }
+    queue_config {
+      capacity             = 5000
+      max_shards           = 10
+      min_shards           = 1
+      max_samples_per_send = 1000
+      batch_send_deadline  = "5s"
+      min_backoff          = "100ms"
+      max_backoff          = "10s"
+    }
+  }
+  wal {
+    truncate_frequency = "30m"
+    min_keepalive_time = "5m"
+    max_keepalive_time = "30m"
+  }
+}
+
+// ----------------------------------------------------------------------------
+// LOGS
+// ----------------------------------------------------------------------------
+
+// systemd journal (host services, ufw, sshd, crowdsec, etc.).
+loki.source.journal "systemd" {
+  path    = "/var/log/journal"
+  max_age = "12h"
+  labels  = {
+    node = sys.env("APEX_NODE_HOST"),
+    job  = "systemd-journal",
+  }
+  forward_to = [loki.write.default.receiver]
+}
+
+// All docker containers' stdout/stderr.
+
+
+
+
+// Push logs to the central Loki.
+loki.write "default" {
+  endpoint {
+    url = sys.env("APEX_LOKI_PUSH_URL")
+    basic_auth {
+      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
+      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
+    }
+  }
+}

etc/alloy/native.alloy

@@ -0,0 +1,128 @@
+// Shared native Grafana Alloy configuration for Docker guests.
+// Metrics and logs use the endpoints and credentials supplied through
+// APEX_PROM_PUSH_URL, APEX_LOKI_PUSH_URL, and APEX_OBS_BASIC_AUTH_USER/PASS.
+
+// ----------------------------------------------------------------------------
+// METRICS
+// ----------------------------------------------------------------------------
+
+// Native filesystem views; physical capacity is collected on the Proxmox host.
+
+
+// Container metrics (cadvisor-equivalent) via the docker socket.
+prometheus.exporter.cadvisor "containers" {
+  docker_host                  = "unix:///var/run/docker.sock"
+  store_container_labels       = false
+  allowlisted_container_labels = [
+    "com.docker.compose.project",
+    "com.docker.compose.service",
+  ]
+  docker_only                  = true
+  enabled_metrics              = [
+    "cpu",
+    "cpuLoad",
+    "diskIO",
+    "memory",
+    "network",
+    "oom_event",
+  ]
+}
+
+// Stamp a static `node` label onto each exporter's targets.
+
+discovery.relabel "container_metrics" {
+  targets = prometheus.exporter.cadvisor.containers.targets
+  rule {
+    action       = "replace"
+    target_label = "node"
+    replacement  = sys.env("APEX_NODE_HOST")
+  }
+  rule {
+    action       = "replace"
+    target_label = "instance"
+    replacement  = sys.env("APEX_NODE_HOST")
+  }
+}
+
+
+prometheus.scrape "containers" {
+  targets         = discovery.relabel.container_metrics.output
+  forward_to      = [prometheus.remote_write.default.receiver]
+  scrape_interval = "30s"
+}
+
+// The configured Prometheus-compatible endpoint must accept remote writes.
+prometheus.remote_write "default" {
+  endpoint {
+    url = sys.env("APEX_PROM_PUSH_URL")
+    basic_auth {
+      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
+      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
+    }
+    queue_config {
+      capacity             = 5000
+      max_shards           = 10
+      min_shards           = 1
+      max_samples_per_send = 1000
+      batch_send_deadline  = "5s"
+      min_backoff          = "100ms"
+      max_backoff          = "10s"
+    }
+  }
+  wal {
+    truncate_frequency = "30m"
+    min_keepalive_time = "5m"
+    max_keepalive_time = "30m"
+  }
+}
+
+// ----------------------------------------------------------------------------
+// LOGS
+// ----------------------------------------------------------------------------
+
+// systemd journal (host services, ufw, sshd, crowdsec, etc.).
+loki.source.journal "systemd" {
+  path    = "/var/log/journal"
+  max_age = "12h"
+  labels  = {
+    node = sys.env("APEX_NODE_HOST"),
+    job  = "systemd-journal",
+  }
+  forward_to = [loki.write.default.receiver]
+}
+
+// All docker containers' stdout/stderr.
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+discovery.relabel "container_logs" {
+  targets = discovery.docker.containers.targets
+  rule {
+    action        = "replace"
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/?(.*)"
+    target_label  = "container"
+  }
+  rule {
+    action       = "replace"
+    target_label = "node"
+    replacement  = sys.env("APEX_NODE_HOST")
+  }
+}
+loki.source.docker "containers" {
+  host          = "unix:///var/run/docker.sock"
+  targets       = discovery.relabel.container_logs.output
+  relabel_rules = discovery.relabel.container_logs.rules
+  forward_to    = [loki.write.default.receiver]
+}
+
+// Push logs to the central Loki.
+loki.write "default" {
+  endpoint {
+    url = sys.env("APEX_LOKI_PUSH_URL")
+    basic_auth {
+      username = sys.env("APEX_OBS_BASIC_AUTH_USER")
+      password = sys.env("APEX_OBS_BASIC_AUTH_PASS")
+    }
+  }
+}