Split host setup into explicit entity actions
This commit is contained in:
51 files changed
+1023
-275
No files matched your search
@@ -0,0 +1,123 @@
|
||||
# actions/tier/backup.py
|
||||
"""Label-driven restic backup via resticontainer (git.ermnvldmr.com/deytenit/resticontainer).
|
||||
Runs the resticontainer core service one-shot: init-if-needed -> backup (discovers
|
||||
`restic.*`-labelled services, runs their pre-hooks/stops, snapshots the union of the
|
||||
resolved host paths in a single restic run) -> forget --prune -> Telegram. Every
|
||||
non-`backup` subcommand passes straight through to restic, so init/snapshots/forget
|
||||
behave exactly as before."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
from engine.descriptor import Meta, Arg
|
||||
|
||||
METADATA = Meta(summary="Run the resticontainer label-driven backup (init-if-needed, backup, forget --prune).",
|
||||
args=[Arg("telegram_bot_url", "Telegram Bot URL for notifications")])
|
||||
TITLE = "Restic Host Backup"
|
||||
|
||||
def write_backup_status(path, state, timestamp):
|
||||
path = Path(path)
|
||||
previous = json.loads(path.read_text()) if path.exists() else {}
|
||||
status = {"state": state, "updated_at": timestamp,
|
||||
"last_success": timestamp if state == "success" else previous.get("last_success", 0)}
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(path.name + f".{os.getpid()}.tmp")
|
||||
temporary.write_text(json.dumps(status) + "\n")
|
||||
temporary.chmod(0o644)
|
||||
temporary.replace(path)
|
||||
|
||||
def restore_containers(ctx, container_ids):
|
||||
"""Recover previously running backup targets, retrying namespace dependents."""
|
||||
pending = list(container_ids)
|
||||
if not pending:
|
||||
return
|
||||
for _ in range(len(pending) + 1):
|
||||
cp = ctx.sys.run(["docker", "inspect", *pending], capture=True)
|
||||
stopped = [c["Id"] for c in json.loads(cp.stdout) if not c["State"]["Running"]]
|
||||
if not stopped:
|
||||
return
|
||||
progress = False
|
||||
for container_id in stopped:
|
||||
cp = ctx.sys.run(["docker", "start", container_id], check=False, capture=True)
|
||||
progress = progress or cp.returncode == 0
|
||||
if not progress:
|
||||
raise RuntimeError("Backup left containers stopped: " + ", ".join(stopped))
|
||||
pending = stopped
|
||||
raise RuntimeError("Backup containers did not stay running: " + ", ".join(pending))
|
||||
|
||||
def run(ctx, args):
|
||||
log, node, url = ctx.log, ctx.node.name, args.telegram_bot_url
|
||||
core = ctx.paths.core
|
||||
host = ctx.node.host # restic --host: the node's public host name (APEX_NODE_HOST)
|
||||
tag = "biweekly" # restic --tag: snapshot cadence (this cron fires on the 5th & 20th)
|
||||
if not os.path.isfile(os.path.join(core, "docker-compose.yml")):
|
||||
log.error(f"Core compose not found in {core}"); raise SystemExit(1)
|
||||
env = dict(os.environ, COMPOSE_PROFILES="manual")
|
||||
env.update({k: v for k, v in ctx.vars().items() if k.startswith("APEX_")}) # tier paths, identity
|
||||
envfiles = []
|
||||
for ef in (".env", "apex.env"): # secrets (.env) + committed scalars (apex.env)
|
||||
if os.path.isfile(os.path.join(core, ef)):
|
||||
envfiles += ["--env-file", ef]
|
||||
|
||||
def restic(*cmd, check=True, capture=False):
|
||||
return ctx.sys.run(["docker", "compose", *envfiles, "--profile", "manual",
|
||||
"run", "--rm", "-T", "resticontainer", *cmd],
|
||||
cwd=core, env=env, check=check, capture=capture)
|
||||
|
||||
def host_snapshot_ids():
|
||||
"""This host's snapshot IDs, or None if it can't be determined (never fail on this)."""
|
||||
cp = restic("snapshots", "--host", host, "--json", check=False, capture=True)
|
||||
if cp.returncode != 0:
|
||||
return None
|
||||
try:
|
||||
return {s.get("id") for s in json.loads(cp.stdout or "[]")}
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
status_path = Path("/var/lib/apex/backup-status.json")
|
||||
write_backup_status(status_path, "running", time.time())
|
||||
try:
|
||||
log.info("Checking restic repository availability...")
|
||||
if restic("snapshots", check=False).returncode == 0:
|
||||
log.info("Restic repository reachable.")
|
||||
elif restic("init", check=False).returncode != 0:
|
||||
ctx.notify.error(TITLE, url, "restic init failed"); raise SystemExit(1)
|
||||
before = host_snapshot_ids()
|
||||
log.info("Backing up `restic.*`-labelled services (resticontainer discovers paths)...")
|
||||
targets = ctx.sys.run(["docker", "ps", "-q", "--no-trunc",
|
||||
"--filter", "label=restic.enable=true",
|
||||
"--filter", "label=restic.backup.stop=true"], capture=True).stdout.split()
|
||||
try:
|
||||
result = restic("backup", "--host", host, "--tag", tag, "--compression", "max",
|
||||
check=False)
|
||||
finally:
|
||||
# Resticontainer may start a network-namespace dependent before its owner.
|
||||
restore_containers(ctx, targets)
|
||||
if result.returncode != 0:
|
||||
ctx.notify.error(TITLE, url, "restic backup failed"); raise SystemExit(1)
|
||||
# Guard: the label-driven backup is a silent no-op if no service carries the
|
||||
# `restic.enable` label (resticontainer finds zero paths and skips restic).
|
||||
# Prove a fresh snapshot for this host actually landed before we prune/notify.
|
||||
after = host_snapshot_ids()
|
||||
if before is not None and after is not None and not (after - before):
|
||||
log.error("No new snapshot produced — refusing to report success.")
|
||||
ctx.notify.error(TITLE, url,
|
||||
"backup produced no new snapshot (no restic.* labels discovered?)")
|
||||
raise SystemExit(1)
|
||||
log.info("Applying retention (forget --prune)...")
|
||||
if restic("forget", "--prune", "--host", host, "--tag", tag,
|
||||
"--keep-daily", "7", "--keep-weekly", "4",
|
||||
"--keep-monthly", "12", check=False).returncode != 0:
|
||||
ctx.notify.error(TITLE, url, "restic forget/prune failed"); raise SystemExit(1)
|
||||
log.success("Restic backup completed.")
|
||||
if not ctx.notify.success(TITLE, url, f"Restic backup + retention succeeded for {node}"):
|
||||
raise SystemExit(1) # bash exited 1 when the success telegram failed to send
|
||||
write_backup_status(status_path, "success", time.time())
|
||||
except SystemExit:
|
||||
write_backup_status(status_path, "failed", time.time())
|
||||
raise
|
||||
except Exception as e:
|
||||
write_backup_status(status_path, "failed", time.time())
|
||||
log.error(f"{type(e).__name__}: {e}")
|
||||
ctx.notify.error(TITLE, url, f"backup failed: {e}")
|
||||
raise SystemExit(1) from e
|
||||
@@ -0,0 +1,76 @@
|
||||
# actions/tier/chown.py
|
||||
"""Port of tier/chown — ownership: {tier}/{proj} -> uid:gid; {tier}/shared/{proj} ->
|
||||
uid:shared_gid (+sticky); base/node-infra -> root:root. Reads APEX_UID/GID from each .env."""
|
||||
import os
|
||||
from engine.descriptor import Meta, Arg
|
||||
|
||||
METADATA = Meta(summary="Apply tier ownership/permissions from each project's APEX_UID/GID/.env.",
|
||||
args=[Arg("tier1_path", "Path to tier 1", required=False),
|
||||
Arg("tier2_path", "Path to tier 2", required=False),
|
||||
Arg("tier3_path", "Path to tier 3", required=False)])
|
||||
SHARED_GROUP = "noroot-shared"
|
||||
|
||||
def _env_val(path, key):
|
||||
if not os.path.exists(path):
|
||||
return ""
|
||||
with open(os.path.realpath(path), encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line.startswith(key + "="):
|
||||
return line.split("=", 1)[1].strip()
|
||||
return ""
|
||||
|
||||
def run(ctx, args):
|
||||
log, s = ctx.log, ctx.sys
|
||||
comp = ctx.paths.compositions
|
||||
# Defaults are the TIER ROOTS (compositions/@tierN, bash $ROOT_TIERn) — NOT
|
||||
# ctx.paths.tierN, which is the apex core project's data dir one level deeper.
|
||||
tiers = [os.path.realpath(args.tier1_path or ctx.paths.root_tier1),
|
||||
os.path.realpath(args.tier2_path or ctx.paths.root_tier2),
|
||||
os.path.realpath(args.tier3_path or ctx.paths.root_tier3)]
|
||||
|
||||
cp = s.run(["getent", "group", SHARED_GROUP], check=False, capture=True)
|
||||
if cp.returncode != 0:
|
||||
log.error(f"Shared group '{SHARED_GROUP}' not found. Run tier/useradd first."); raise SystemExit(1)
|
||||
|
||||
log.info("Ensuring base structures owned by root...")
|
||||
for tp in tiers:
|
||||
s.sudo(["chown", "root:root", tp])
|
||||
shared = os.path.join(tp, "shared")
|
||||
if os.path.isdir(shared): # dir inode only — recursing would clobber
|
||||
s.sudo(["chown", "root:root", shared]) # skipped projects' shared subtrees
|
||||
infra = os.path.join(tp, "node-infra")
|
||||
if os.path.isdir(infra):
|
||||
s.sudo(["chown", "-R", "root:root", infra])
|
||||
|
||||
for proj in sorted(os.listdir(comp)):
|
||||
full = os.path.join(comp, proj)
|
||||
if not os.path.isdir(full) or proj.startswith(".") or proj.startswith("@"):
|
||||
continue
|
||||
# opt-out: .apex-root-owned forces this project's tier data (dir + children,
|
||||
# recursively) to root:root — for services that run as root and reject
|
||||
# non-root-owned data (e.g. seafile-mc). Blocks noroot ownership entirely.
|
||||
if os.path.isfile(os.path.join(full, ".apex-root-owned")):
|
||||
log.info(f"Applying root ownership for {proj} (.apex-root-owned)")
|
||||
for tp in tiers:
|
||||
for d in (os.path.join(tp, proj), os.path.join(tp, "shared", proj)):
|
||||
if os.path.isdir(d):
|
||||
s.sudo(["chown", "-R", "root:root", d]); log.success(f" Owned {d} (root)")
|
||||
continue
|
||||
env_path = os.path.join(full, ".env")
|
||||
uid = _env_val(env_path, "APEX_UID")
|
||||
gid = _env_val(env_path, "APEX_GID")
|
||||
sgid = _env_val(env_path, "APEX_SHARED_GID")
|
||||
if not (uid and gid and sgid):
|
||||
log.warn(f"Incomplete APEX_UID/GID/SHARED_GID for {proj}. Run tier/useradd first. Skipping.")
|
||||
continue
|
||||
log.info(f"Applying permissions for {proj} (UID={uid} GID={gid} SHARED_GID={sgid})")
|
||||
for tp in tiers:
|
||||
pdir = os.path.join(tp, proj)
|
||||
sdir = os.path.join(tp, "shared", proj)
|
||||
if os.path.isdir(pdir):
|
||||
s.sudo(["chown", "-R", f"{uid}:{gid}", pdir]); log.success(f" Owned {pdir}")
|
||||
if os.path.isdir(sdir):
|
||||
s.sudo(["chown", "-R", f"{uid}:{sgid}", sdir])
|
||||
s.sudo(["chmod", "+t", sdir]); log.success(f" Owned {sdir} (sticky)")
|
||||
log.success("Permissions applied.")
|
||||
@@ -0,0 +1,69 @@
|
||||
# actions/tier/link.py
|
||||
"""Port of tier/link — @tier symlink structure for the node + each composition project."""
|
||||
import os
|
||||
from engine.descriptor import Meta, Arg
|
||||
|
||||
METADATA = Meta(summary="Create @tier symlink structure for the node + each composition project.",
|
||||
args=[Arg("tier1_path", "Path to tier 1 storage root"),
|
||||
Arg("tier2_path", "Path to tier 2 storage root"),
|
||||
Arg("tier3_path", "Path to tier 3 storage root")])
|
||||
|
||||
def run(ctx, args):
|
||||
log, s = ctx.log, ctx.sys
|
||||
comp = ctx.paths.compositions
|
||||
tiers = [os.path.realpath(args.tier1_path),
|
||||
os.path.realpath(args.tier2_path),
|
||||
os.path.realpath(args.tier3_path)]
|
||||
|
||||
def confirm(msg):
|
||||
try:
|
||||
return input(f"{msg} [y/N]: ").strip().lower() in ("y", "yes")
|
||||
except EOFError: # non-interactive stdin = decline, like bash `read` at EOF
|
||||
return False
|
||||
|
||||
def safe_remove(link, desc):
|
||||
if os.path.islink(link):
|
||||
s.sudo(["rm", link]); return True
|
||||
if os.path.exists(link):
|
||||
log.warn(f"Path exists but is not a symlink: {link}")
|
||||
if confirm(f"Remove existing {desc} (NOT a symlink)?"):
|
||||
s.sudo(["rm", "-rf", link]); return True
|
||||
return False
|
||||
return True
|
||||
|
||||
def mklink(target, link, desc):
|
||||
s.sudo(["ln", "-snf", target, link])
|
||||
log.success(f"Created {desc} symlink: {link} -> {target}")
|
||||
|
||||
log.info("Starting @tier setup...")
|
||||
for t in tiers:
|
||||
s.sudo(["mkdir", "-p", t])
|
||||
# node-level @tierN -> tier roots
|
||||
for i, n in enumerate((1, 2, 3)):
|
||||
link = os.path.join(comp, f"@tier{n}")
|
||||
if safe_remove(link, f"node-@tier{n}"):
|
||||
mklink(tiers[i], link, f"node-@tier{n}")
|
||||
# per-project (every composition subdir except dot/@)
|
||||
for proj in sorted(os.listdir(comp)):
|
||||
full = os.path.join(comp, proj)
|
||||
if not os.path.isdir(full) or proj.startswith(".") or proj.startswith("@"):
|
||||
continue
|
||||
log.info(f"Processing project: {proj}")
|
||||
for i, n in enumerate((1, 2, 3)):
|
||||
base = tiers[i]
|
||||
s.sudo(["mkdir", "-p", os.path.join(base, "shared")])
|
||||
s.sudo(["mkdir", "-p", os.path.join(base, "shared", proj)])
|
||||
s.sudo(["mkdir", "-p", os.path.join(base, proj)])
|
||||
shared_link = os.path.join(base, proj, "shared")
|
||||
if safe_remove(shared_link, "shared-internal"):
|
||||
s.sudo(["ln", "-snf", "../shared", shared_link])
|
||||
link = os.path.join(full, f"@tier{n}")
|
||||
if safe_remove(link, f"@tier{n}"):
|
||||
mklink(os.path.join(base, proj), link, f"@tier{n}")
|
||||
env_link = os.path.join(full, ".env")
|
||||
env_target = os.path.join(tiers[0], proj, ".env")
|
||||
if safe_remove(env_link, ".env"):
|
||||
if not os.path.isfile(env_target):
|
||||
s.sudo(["touch", env_target])
|
||||
mklink(env_target, env_link, ".env")
|
||||
log.success("@tier setup completed.")
|
||||
@@ -0,0 +1,82 @@
|
||||
# actions/tier/useradd.py
|
||||
"""Port of tier/useradd — one noroot system user per composition project; inject the
|
||||
APEX_UID/APEX_GID/APEX_SHARED_GID block into each project .env."""
|
||||
import os, re
|
||||
from engine.descriptor import Meta
|
||||
|
||||
METADATA = Meta(summary="Create a noroot system user per composition project; write APEX_UID/GID into .env.")
|
||||
SHARED_GROUP = "noroot-shared"
|
||||
START = "# START AUTO GENERATED APEX-NOROOT-USERS"
|
||||
END = "# END AUTO GENERATED APEX-NOROOT-USERS"
|
||||
|
||||
def _sanitize(raw):
|
||||
s = re.sub(r"-+$", "", re.sub(r"^-+", "", re.sub(r"[^a-z0-9]+", "-", raw.lower())))
|
||||
s = re.sub(r"-+", "-", s) or "project"
|
||||
return re.sub(r"-+$", "", s[:24])
|
||||
|
||||
def run(ctx, args):
|
||||
log, sysx = ctx.log, ctx.sys
|
||||
comp = ctx.paths.compositions
|
||||
|
||||
def getent_group(name):
|
||||
cp = sysx.run(["getent", "group", name], check=False, capture=True)
|
||||
return cp.stdout.split(":")[2].strip() if cp.returncode == 0 else ""
|
||||
|
||||
shared_gid = getent_group(SHARED_GROUP)
|
||||
if not shared_gid:
|
||||
log.info(f"Creating shared group: {SHARED_GROUP}")
|
||||
sysx.sudo(["groupadd", "-r", SHARED_GROUP])
|
||||
shared_gid = getent_group(SHARED_GROUP)
|
||||
log.info(f"Shared group {SHARED_GROUP} (GID {shared_gid})")
|
||||
|
||||
used = {}
|
||||
for proj in sorted(os.listdir(comp)):
|
||||
full = os.path.join(comp, proj)
|
||||
if not os.path.isdir(full) or proj.startswith(".") or proj.startswith("@"):
|
||||
continue
|
||||
# opt-out: a project marked .apex-root-owned runs as root and keeps its tier
|
||||
# data root-owned (e.g. seafile-mc, whose entrypoint refuses non-root-owned
|
||||
# data). Skip the noroot user + .env UID injection; tier/chown forces root.
|
||||
if os.path.isfile(os.path.join(full, ".apex-root-owned")):
|
||||
log.info(f"{proj}: .apex-root-owned — runs as root; skipping noroot user")
|
||||
continue
|
||||
base = f"noroot-{_sanitize(proj)}"
|
||||
cand, i = base, 1
|
||||
while cand in used:
|
||||
cand = f"{base}-{i}"; i += 1
|
||||
used[cand] = proj
|
||||
exists = sysx.run(["getent", "passwd", cand], check=False, capture=True)
|
||||
if exists.returncode != 0:
|
||||
sysx.sudo(["useradd", "-r", "-M", "-d", "/nonexistent", "-s", "/usr/sbin/nologin",
|
||||
"-c", f"System user for project {proj}", "-U", cand])
|
||||
sysx.run(["bash", "-c", f"sudo passwd -l {cand} >/dev/null 2>&1 || true"])
|
||||
log.success(f"Created system user {cand} (project {proj})")
|
||||
else:
|
||||
log.info(f"User {cand} exists (project {proj})")
|
||||
if not sysx.ok(["bash", "-c", f"groups {cand} 2>/dev/null | grep -q '\\b{SHARED_GROUP}\\b'"]):
|
||||
sysx.sudo(["usermod", "-aG", SHARED_GROUP, cand])
|
||||
pw = sysx.run(["getent", "passwd", cand], check=False, capture=True)
|
||||
parts = pw.stdout.split(":") if pw.returncode == 0 else []
|
||||
uid, gid = (parts[2], parts[3]) if len(parts) >= 4 else ("", "")
|
||||
env_path = os.path.join(full, ".env")
|
||||
if uid and gid and shared_gid and os.path.exists(env_path):
|
||||
_inject_env(ctx, env_path, uid, gid, shared_gid)
|
||||
elif os.path.exists(env_path):
|
||||
log.warn(f"Could not determine UID/GID for {cand}; skipping .env update")
|
||||
else:
|
||||
log.info(f"No .env present for project {proj}")
|
||||
log.success("noroot user provisioning completed.")
|
||||
|
||||
def _inject_env(ctx, env_path, uid, gid, shared_gid):
|
||||
actual = os.path.realpath(env_path)
|
||||
with open(actual) as f:
|
||||
text = f.read()
|
||||
block = f"{START}\nAPEX_UID={uid}\nAPEX_GID={gid}\nAPEX_SHARED_GID={shared_gid}\n{END}\n"
|
||||
if START in text:
|
||||
new = re.sub(re.escape(START) + r".*?" + re.escape(END) + r"\n", block, text, flags=re.S)
|
||||
else:
|
||||
new = block + "\n" + text
|
||||
# backup=False: bash never left copies; a timestamped .bak would pile up a
|
||||
# secret-bearing .env duplicate per run next to the bind-mounted project dir.
|
||||
ctx.host.write_file(actual, new, backup=False, sudo=True)
|
||||
ctx.log.success(f"Updated {actual} with APEX_UID/GID/SHARED_GID")
|
||||
Reference in new issue
Block a user