Remove the image build and consolidate operational code under scripts

This commit is contained in:
deytenit committed 2026-11-08 12:39:49 +03:00
1 parent ef5be2331d
commit 860ff34c70
77 files changed
+586 -260607

No files matched your search

View File
Whitespace-only changes.
+28
View File
@@ -0,0 +1,28 @@
# scripts/engine/lib/crowdsec.py
"""ctx.crowdsec — install + config deploy + collections (port of config/crowdsec)."""
from __future__ import annotations
import os
import shutil
class Crowdsec:
def __init__(self, log, sys_, tpl, host):
self.log, self.sys, self.tpl, self.host = log, sys_, tpl, host
def ensure_installed(self) -> None:
if shutil.which("cscli"):
return
self.log.info("Installing CrowdSec via official script...")
self.sys.run(["bash", "-c", "set -o pipefail; curl -sSfL https://install.crowdsec.net | sudo sh"],
check=True)
self.sys.sudo(["apt-get", "install", "-y", "crowdsec"])
def deploy(self, configs_subdir: str, vars: dict, dry_run: bool = False) -> None:
for rel, content in self.tpl.render_dir(configs_subdir, vars).items():
self.host.write_file(os.path.join("/etc/crowdsec", rel), content,
backup=True, dry_run=dry_run, sudo=True)
def install_collections(self, collections) -> None:
self.sys.sudo(["cscli", "hub", "update"])
for c in collections:
self.sys.sudo(["cscli", "collections", "install", c])
+191
View File
@@ -0,0 +1,191 @@
# scripts/engine/lib/host.py
"""ctx.host — idempotent host mutation: backup, write_file, inject_block, install_crontab.
Ports root_template_crontab (install + rollback) and the ufw block-injection logic
(root_template.sh _root_template_inject_rules_to_ufw_file), generalized + self-healing.
"""
from __future__ import annotations
import inspect
import json
import os
import subprocess
import time
from typing import List, Optional, Sequence, Tuple
def apply_block(text: str, content: str, start: str, end: str,
anchor: Optional[str] = None,
old_pairs: Sequence[Tuple[str, str]] = ()) -> Tuple[str, bool]:
"""Pure transform: remove existing (start,end) and old_pairs blocks, insert a fresh
block after the first anchor line (or append if anchor is None). If an anchor is
given but not present, return (text, False) unchanged (matches bash skip behavior)."""
lines = text.splitlines()
if anchor is not None and not any(anchor in ln for ln in lines):
return text, False
pairs = [(start, end), *old_pairs]
stripped_starts = {s: e for s, e in pairs}
out: List[str] = []
skip_until: Optional[str] = None
for ln in lines:
if skip_until is None:
s = ln.strip()
if s in stripped_starts:
skip_until = stripped_starts[s]
continue
out.append(ln)
else:
if ln.strip() == skip_until:
skip_until = None
continue
block = [start, *content.splitlines(), end]
if anchor is None:
out.extend(block)
return "\n".join(out) + "\n", True
result: List[str] = []
inserted = False
for ln in out:
result.append(ln)
if not inserted and anchor in ln:
result.extend(block)
inserted = True
return "\n".join(result) + "\n", inserted
def _atomic_write(path, content, mode):
"""Self-contained so sudo executes the same operation without importing user code."""
import os
import stat
import tempfile
target = os.path.realpath(path)
parent = os.path.dirname(target)
try:
previous = os.stat(target)
except FileNotFoundError:
previous = None
if previous is not None and not stat.S_ISREG(previous.st_mode):
raise ValueError(f"Not a regular file: {path}")
target_mode = (int(mode, 8) if mode is not None else
stat.S_IMODE(previous.st_mode) if previous is not None else 0o644)
os.makedirs(parent, exist_ok=True)
fd, temporary = tempfile.mkstemp(prefix=".apex-", dir=parent)
try:
with os.fdopen(fd, "w", encoding="utf-8", newline="") as stream:
stream.write(content)
stream.flush()
if previous is not None:
current = os.fstat(stream.fileno())
if (current.st_uid, current.st_gid) != (previous.st_uid, previous.st_gid):
os.fchown(stream.fileno(), previous.st_uid, previous.st_gid)
os.fchmod(stream.fileno(), target_mode)
os.fsync(stream.fileno())
os.replace(temporary, target)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
def _atomic_backup(path, destination):
import os
import shutil
import tempfile
destination = os.path.abspath(destination)
fd, temporary = tempfile.mkstemp(prefix=".apex-", dir=os.path.dirname(destination))
os.close(fd)
try:
shutil.copyfile(path, temporary)
previous, current = os.stat(path), os.stat(temporary)
if (current.st_uid, current.st_gid) != (previous.st_uid, previous.st_gid):
os.chown(temporary, previous.st_uid, previous.st_gid)
shutil.copystat(path, temporary)
os.replace(temporary, destination)
finally:
if os.path.exists(temporary):
os.unlink(temporary)
class Host:
def __init__(self, log, sys_):
self.log = log
self.sys = sys_
def _file_operation(self, operation, args, sudo):
if sudo:
script = (inspect.getsource(operation) + "\nimport json, sys\n" +
f"{operation.__name__}(*json.load(sys.stdin))\n")
self.sys.sudo(["python3", "-I", "-c", script], input=json.dumps(args))
else:
operation(*args)
# --- read/write with sudo awareness ---
def _read(self, path: str, sudo: bool = False) -> str:
if sudo:
cp = self.sys.sudo(["cat", path], check=True, capture=True)
return cp.stdout or ""
with open(path, encoding="utf-8") as f:
return f.read()
def backup(self, path: str, sudo: bool = False) -> Optional[str]:
exists = (self.sys.sudo(["test", "-f", path], check=False).returncode == 0 if sudo
else os.path.isfile(path))
if not exists:
return None
bkp = f"{path}.{int(time.time())}.bak"
self._file_operation(_atomic_backup, [path, bkp], sudo)
self.log.info(f"Backed up {path} -> {bkp}")
return bkp
def write_file(self, path: str, content: str, mode: Optional[str] = None,
backup: bool = True, dry_run: bool = False, sudo: bool = False) -> None:
if dry_run:
self.log.info(f"[DRY-RUN] would write {len(content)} bytes to {path}")
return
if backup:
self.backup(path, sudo=sudo)
self._file_operation(_atomic_write, [path, content, mode], sudo)
self.log.success(f"Wrote {path}")
def inject_block(self, target: str, content: str, start: str, end: str,
anchor: Optional[str] = None,
old_pairs: Sequence[Tuple[str, str]] = (),
dry_run: bool = False, sudo: bool = True) -> bool:
exists = (self.sys.sudo(["test", "-f", target], check=False).returncode == 0 if sudo
else os.path.isfile(target))
if not exists:
self.log.error(f"Target file does not exist: {target}")
return False
text = self._read(target, sudo=sudo)
new_text, injected = apply_block(text, content, start, end, anchor, old_pairs)
if not injected:
self.log.warn(f"Anchor {anchor!r} not found in {target}; skipping.")
return False
if dry_run:
self.log.info(f"[DRY-RUN] would inject block into {target}")
return True
self.write_file(target, new_text, backup=True, sudo=sudo)
self.log.success(f"Block injected into {target}")
return True
def install_crontab(self, text: str, dry_run: bool = False) -> None:
"""Port of root_template_crontab: backup, install via `crontab -`, rollback."""
backup = subprocess.run(["crontab", "-l"], text=True,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
prev = backup.stdout if backup.returncode == 0 else ""
if not text.strip():
self.log.error("Rendered crontab is empty; aborting (previous kept).")
raise SystemExit(65) # EX_DATAERR
self.log.info("Rendered crontab preview:\n" + text)
if dry_run:
self.log.info("[DRY-RUN] would install the crontab above.")
return
res = subprocess.run(["crontab", "-"], input=text, text=True)
if res.returncode != 0:
self.log.error("crontab install failed; restoring previous.")
subprocess.run(["crontab", "-"], input=prev, text=True)
raise SystemExit(1)
self.log.success("Crontab installed.")
+19
View File
@@ -0,0 +1,19 @@
# scripts/engine/lib/log.py
"""ctx.log — [node] [action] [level] message (port of root_log_*)."""
from __future__ import annotations
import sys
class Log:
def __init__(self, node: str = "global", action: str = "core"):
self.node = node
self.action = action
def _emit(self, level: str, msg: str, stream=None) -> None:
print(f"[{self.node}] [{self.action}] [{level}] {msg}",
file=stream or sys.stdout, flush=True)
def info(self, msg): self._emit("INFO", msg)
def warn(self, msg): self._emit("WARN", msg)
def error(self, msg): self._emit("ERROR", msg, sys.stderr)
def success(self, msg): self._emit("SUCCESS", msg)
+87
View File
@@ -0,0 +1,87 @@
# scripts/engine/lib/notify.py
"""ctx.notify — Telegram notifications.
The alert `instance` is the node's public host (APEX_NODE_HOST), passed in from the
context. POSTs application/x-www-form-urlencoded via urllib (stdlib).
"""
from __future__ import annotations
import html
import urllib.parse
import urllib.request
from datetime import datetime
NOTIFY_TIMEOUT = 20
MAX_MESSAGE_LEN = 3500
def resolve_bot_url(value: str) -> str:
"""Resolve a literal URL or an @file reference without putting secrets in argv."""
if not value or not value.startswith("@"):
return value
path = value[1:]
if not path:
raise ValueError("empty Telegram Bot URL file reference")
with open(path, encoding="utf-8") as f:
return f.read().strip()
def build_text(title: str, message: str, node: str, level: str, ts: str) -> str:
status, icon, severity = "firing", "🚨", "info"
if level == "SUCCESS":
status, icon = "resolved", "✅"
elif level == "ERROR":
severity = "critical"
elif level == "WARN":
severity = "warning"
instance = node
msg = message if len(message) <= MAX_MESSAGE_LEN else message[:MAX_MESSAGE_LEN] + "..."
msg = html.escape(msg)
safe_title = html.escape(title)
return (f"{icon} <b>{safe_title}</b>\n"
f"<b>Instance:</b> {instance}\n"
f"<b>Timestamp:</b> {ts}\n"
f"<b>Status:</b> {status.capitalize()}\n"
f"<b>Escalation:</b> {severity}\n"
f"<pre><code>{msg}</code></pre>\n"
f"#status_{status} #escalation_{severity} #instance_{instance.replace('.', '_')}")
class Notify:
def __init__(self, log, node: str = "global"):
self.log = log
self.node = node
def telegram(self, title, bot_url, message, node=None, level="INFO") -> bool:
node = node or self.node
try:
bot_url = resolve_bot_url(bot_url)
except (OSError, ValueError) as e:
self.log.error(f"Failed to read Telegram Bot URL: {e}")
return False
if not bot_url:
self.log.warn("Telegram Bot URL not provided. Skipping notification.")
return False
parsed = urllib.parse.urlparse(bot_url)
if parsed.scheme != "https":
self.log.error(f"Telegram Bot URL must use HTTPS, got scheme: {parsed.scheme!r}")
return False
ts = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")
text = build_text(title, message, node, level, ts)
data = urllib.parse.urlencode({"parse_mode": "HTML", "text": text}).encode()
self.log.info(f"Sending Telegram notification ({level}): {title}")
try:
with urllib.request.urlopen(urllib.request.Request(bot_url, data=data), # noqa: S310 -- HTTPS validated above.
timeout=NOTIFY_TIMEOUT) as r:
r.read()
return True
except Exception as e:
self.log.error(f"Failed to send Telegram notification: {e}")
return False
def success(self, title, url, msg, node=None): return self.telegram(title, url, msg, node, "SUCCESS")
def error(self, title, url, msg, node=None): return self.telegram(title, url, msg, node, "ERROR")
def info(self, title, url, msg, node=None): return self.telegram(title, url, msg, node, "INFO")
def warn(self, title, url, msg, node=None): return self.telegram(title, url, msg, node, "WARN")
+98
View File
@@ -0,0 +1,98 @@
# scripts/engine/lib/sys.py
"""ctx.sys — command execution + systemd service management (port of root_sys.sh)."""
from __future__ import annotations
import os
import shutil
import subprocess
import time
from typing import List, Optional, Sequence
class Sys:
def __init__(self, log):
self.log = log
# --- command execution ---
def run(self, cmd: Sequence[str], check: bool = True, capture: bool = False,
cwd: Optional[str] = None, env: Optional[dict] = None,
input: Optional[str] = None) -> subprocess.CompletedProcess:
return subprocess.run(list(cmd), check=check, cwd=cwd, env=env, text=True,
input=input,
stdout=subprocess.PIPE if capture else None,
stderr=subprocess.PIPE if capture else None)
def sudo(self, cmd: Sequence[str], **kw) -> subprocess.CompletedProcess:
if os.geteuid() == 0 and shutil.which("sudo") is None:
return self.run(cmd, **kw)
return self.run(["sudo", *cmd], **kw)
def ok(self, cmd: Sequence[str], **kw) -> bool:
"""Run, return True on exit 0, never raising."""
return self.run(cmd, check=False, **kw).returncode == 0
# --- systemd (port of root_sys_*) ---
def service_exists(self, name: str) -> bool:
cp = self.run(["systemctl", "list-unit-files", "--type=service"],
check=False, capture=True)
return any(line.startswith(f"{name}.service")
for line in (cp.stdout or "").splitlines())
def service_is_active(self, name: str) -> bool:
return self.ok(["systemctl", "is-active", "--quiet", name])
def service_is_enabled(self, name: str) -> bool:
return self.ok(["systemctl", "is-enabled", "--quiet", name])
def start_and_enable(self, name: str) -> bool:
if not self.service_exists(name):
self.log.warn(f"Service {name} does not exist on this system")
return False
if not self.service_is_active(name):
self.log.info(f"Starting {name} service...")
self.sudo(["systemctl", "start", name])
if not self.service_is_enabled(name):
self.log.info(f"Enabling {name} service...")
self.sudo(["systemctl", "enable", name])
return self.service_is_active(name)
def ensure_running(self, name: str) -> None:
"""Port of root_init_ensure_service_running: enable --now if not active."""
if not self.service_is_active(name):
self.log.info(f"Starting {name} service...")
self.sudo(["systemctl", "enable", "--now", name])
else:
self.log.info(f"{name} service already running.")
def restart(self, name: str) -> bool:
self.log.info(f"Restarting {name} service...")
self.sudo(["systemctl", "restart", name])
return self.service_is_active(name)
def reload(self, name: str, dry_run: bool = False) -> None:
if dry_run:
self.log.info(f"[DRY-RUN] systemctl reload {name}")
return
self.log.info(f"Reloading {name} configuration...")
self.sudo(["systemctl", "reload", name])
def daemon_reload(self) -> None:
self.sudo(["systemctl", "daemon-reload"])
def wait_for(self, name: str, timeout: int = 30, interval: int = 2) -> bool:
elapsed = 0
while elapsed < timeout:
if self.service_is_active(name):
return True
time.sleep(interval)
elapsed += interval
return False
def install_packages(self, packages: List[str]) -> None:
"""Port of root_init_install_packages (install if the binary is absent)."""
for pkg in packages:
if shutil.which(pkg):
self.log.info(f"{pkg} already installed.")
continue
self.log.info(f"Installing {pkg}...")
self.sudo(["apt-get", "update", "-y"])
self.sudo(["apt-get", "install", "-y", pkg])
+26
View File
@@ -0,0 +1,26 @@
# scripts/engine/lib/systemd.py
"""ctx.systemd — render + install unit files, daemon-reload, enable+start."""
from __future__ import annotations
import os
class Systemd:
def __init__(self, log, sys_, tpl, host):
self.log, self.sys, self.tpl, self.host = log, sys_, tpl, host
def deploy_units(self, configs_subdir: str, vars: dict, dry_run: bool = False) -> None:
# Top-level, non-hidden files only — the bash glob ("$CONFIG_DIR"/* + [[ -f ]])
# never recursed nor picked up dotfiles like .gitkeep.
rendered = {rel: c for rel, c in self.tpl.render_dir(configs_subdir, vars).items()
if "/" not in rel and not rel.startswith(".")}
if not rendered:
self.log.warn(f"No systemd units found under configs/{configs_subdir}.")
return
for rel, content in sorted(rendered.items()):
name = os.path.basename(rel)
self.host.write_file(f"/etc/systemd/system/{name}", content,
backup=True, dry_run=dry_run, sudo=True)
if name.endswith(".service") and not dry_run:
self.sys.daemon_reload()
self.sys.ensure_running(name[: -len(".service")])
self.log.success(f"Service {name[:-8]} active and enabled.")
+37
View File
@@ -0,0 +1,37 @@
# scripts/engine/lib/tpl.py
"""ctx.tpl — stdlib string.Template rendering.
safe_substitute replaces known $VAR/${VAR} and leaves unknown ones literal.
Shell $(...) command-substitution survives untouched.
Loops/conditionals live in the python action, never in the template file.
"""
from __future__ import annotations
import os
from string import Template
from typing import Dict
class Tpl:
def __init__(self, configs_dir: str, log):
self.configs_dir = configs_dir
self.log = log
def _resolve(self, path: str) -> str:
return path if os.path.isabs(path) else os.path.join(self.configs_dir, path)
def render(self, path: str, vars: Dict[str, str]) -> str:
full = self._resolve(path)
with open(full) as f:
return Template(f.read()).safe_substitute(vars)
def render_dir(self, rel_dir: str, vars: Dict[str, str]) -> Dict[str, str]:
"""Render every file under configs/<rel_dir>. Returns {relpath: content}."""
base = self._resolve(rel_dir)
out: Dict[str, str] = {}
for dirpath, _dirs, files in os.walk(base):
for fn in sorted(files):
full = os.path.join(dirpath, fn)
rel = os.path.relpath(full, base).replace(os.sep, "/")
with open(full) as f:
out[rel] = Template(f.read()).safe_substitute(vars)
return out
+105
View File
@@ -0,0 +1,105 @@
# scripts/engine/lib/ufw.py
"""Apply ufw and ufw-docker rules.
Replace existing supported marker blocks before inserting current rules.
Docker rules go through ufw-docker; host rules through plain ufw.
"""
from __future__ import annotations
import os
import shlex
import shutil
APEX_START = "# START APEX.ERMNVLDMR.COM RULES"
APEX_END = "# END APEX.ERMNVLDMR.COM RULES"
OLD_PAIRS = [("# START ROOT.ERMNVLDMR.COM RULES", "# END ROOT.ERMNVLDMR.COM RULES")]
def resolve_ufw_docker_bin() -> str:
"""Find ufw-docker in PATH, /usr/local/bin, or fallback to ~/.local/bin/ufw-docker."""
found = shutil.which("ufw-docker")
if found:
return found
if os.path.isfile("/usr/local/bin/ufw-docker") and os.access("/usr/local/bin/ufw-docker", os.X_OK):
return "/usr/local/bin/ufw-docker"
return os.path.expanduser("~/.local/bin/ufw-docker")
UFW_DOCKER_BIN = resolve_ufw_docker_bin()
def clean_rules(text: str):
"""Strip whitespace, drop blank + comment lines (matches the bash awk filter)."""
return [s for s in (ln.strip() for ln in text.splitlines()) if s and not s.startswith("#")]
class Ufw:
def __init__(self, log, sys_, host):
self.log, self.sys, self.host = log, sys_, host
def _inject_file(self, target: str, source_file: str, dry_run: bool):
if not os.path.isfile(source_file):
return
if not self.sys.ok(["sudo", "test", "-f", target]):
self.log.error(f"Target file does not exist: {target}")
raise SystemExit(1)
with open(source_file, encoding="utf-8") as f:
content = "\n".join(clean_rules(f.read()))
self.host.inject_block(
target,
content,
APEX_START,
APEX_END,
anchor="# End required lines",
old_pairs=OLD_PAIRS,
dry_run=dry_run,
sudo=True,
)
def apply(self, rules_dir: str | None = None, config_dir: str | None = None, dry_run: bool = False) -> None:
if rules_dir and not os.path.isdir(rules_dir):
self.log.error(f"Rules directory does not exist: {rules_dir}")
raise SystemExit(66)
if config_dir and not os.path.isdir(config_dir):
self.log.error(f"Config directory does not exist: {config_dir}")
raise SystemExit(66)
if not rules_dir and not config_dir:
self.log.warn("Neither docker nor host ufw rules directory found; nothing to apply.")
return
if not self.sys.ok(["sudo", "ufw", "--version"]):
self.log.error("ufw not installed/working via sudo. Run config/packages first.")
raise SystemExit(1)
if not (os.path.isfile(UFW_DOCKER_BIN) and os.access(UFW_DOCKER_BIN, os.X_OK)):
self.log.error(f"ufw-docker not found at {UFW_DOCKER_BIN}. Run config/ufw-docker first.")
raise SystemExit(1)
if config_dir:
host_rules = os.path.join(config_dir, "host.rules")
if os.path.isfile(host_rules):
with open(host_rules, encoding="utf-8") as f:
rules = clean_rules(f.read())
for rule in rules:
self.log.info(f"Applying host ufw rule: {rule}")
if dry_run:
self.log.info(f"[DRY-RUN] sudo ufw {rule}")
else:
self.sys.sudo(["ufw", *shlex.split(rule)], check=True)
self._inject_file("/etc/ufw/before.rules", os.path.join(config_dir, "before.rules"), dry_run)
self._inject_file("/etc/ufw/after.rules", os.path.join(config_dir, "after.rules"), dry_run)
if rules_dir:
rule_files = [fn for fn in sorted(os.listdir(rules_dir)) if fn.endswith(".rules") and not fn.startswith(".")]
if not rule_files:
self.log.warn(f"No .rules files found in {rules_dir}, nothing to apply.")
for fn in rule_files:
with open(os.path.join(rules_dir, fn), encoding="utf-8") as f:
rules = clean_rules(f.read())
for rule in rules:
self.log.info(f"Applying ufw-docker rule ({fn}): {rule}")
if dry_run:
self.log.info(f"[DRY-RUN] sudo {UFW_DOCKER_BIN} {rule}")
else:
self.sys.sudo([UFW_DOCKER_BIN, *shlex.split(rule)], check=True)
self.log.success("All ufw rules applied.")
self.log.warn("Run `sudo systemctl restart ufw` to apply. Ensure ssh is allowed via plain ufw!")