diff --git a/PARITY.md b/PARITY.md index 167ea23..7860e19 100644 --- a/PARITY.md +++ b/PARITY.md @@ -14,7 +14,7 @@ action to its bash source and the behaviors that must match. | configure | (new umbrella) | run base,ufw,crowdsec,cron,systemd,routing in order | | sync/repository | actions/sync/repository | sync/ branch; add -A + commons submodule; commit; rebase origin/main (was trunk in the monorepo); force-with-lease; telegram | | sync/packages | actions/sync/packages | apt --just-print upgrade Inst list; docker+skopeo digest compare; telegram info | -| backup/run | actions/sync/tiers (restic-backup) | init-if-needed; backup /data/@tier1,2 --host --tag com-ermnvldmr-root-; forget --prune 7/4/12; telegram | +| backup/run | actions/sync/tiers (restic-backup) | init-if-needed; label-driven backup via resticontainer (discovers `restic.*`-labelled services, runs pre-hooks/stops, snapshots the resolved host paths) --host .apex.ermnvldmr.com --tag biweekly --compression max; guard on a fresh snapshot; forget --prune 7/4/12; telegram | | tiers/link | actions/tiers/link | node + per-project @tierN symlinks; shared/ links; .env symlink | | tiers/useradd | actions/tiers/useradd | noroot- system users; noroot-shared group; APEX_UID/GID block in .env | | tiers/chown | actions/tiers/chown | {tier}/{proj}=uid:gid; {tier}/shared/{proj}=uid:sgid+sticky; base=root:root | @@ -26,7 +26,8 @@ action to its bash source and the behaviors that must match. ## Deliberate deltas (rename + fixes) - root->apex everywhere (CLI, container names apex-*, telegram instance com.ermnvldmr.apex.*, ufw markers APEX.*, networks direct/enclave/socket). - No arg; identity from FQDN + node.env. -- restic core service gains read-only /data/@tier1,2 mounts (makes existing backup command functional). +- backup superseded by the resticontainer migration (see below): the old whole-tier-root + restic snapshot is replaced by per-service, `restic.*`-label-driven backups. - ufw markers self-heal (delete ROOT + APEX blocks before inject). - envsubst -> string.Template.safe_substitute; wget/curl download -> urllib. Unset template vars stay LITERAL (envsubst emptied them) — typos surface instead of silently blanking. @@ -57,3 +58,18 @@ action to its bash source and the behaviors that must match. - restic: per-node selective tier2 mounts replaced by whole-tier-root RO mounts (superset; snapshot paths preserved); RESTIC_COMPRESSION per-node via `APEX_RESTIC_COMPRESSION` (default auto). + +## resticontainer migration (SP4 realized) +- The `restic` core service is replaced by `resticontainer` + (`ghcr.io/deytenit/resticontainer`), a restic wrapper that reads per-service + `restic.*` compose labels: `restic.enable`, `restic.backup.paths` (comma-separated + container mount destinations — matched exactly, whole mount dir backed up), + `restic.backup.stop`, and `restic.hooks.pre-backup`/`post-backup` (run via docker exec). +- It mounts the docker socket (list/inspect/exec/stop/start) and the host root + `/:/hostfs:ro` (`RESTIC_HOSTFS=/hostfs`); it resolves each labelled mount to + `/hostfs` + the mount's host source and snapshots the union in one restic run. +- The whole-tier-root RO mounts and the explicit `/data/@tier1,2` backup args are gone; + a service is backed up only if it carries `restic.enable=true`. DB consistency is + per-service: a pre-backup hook dumps into a dedicated backed-up mount (e.g. postgres + `pg_dump … > /dumps/dump.sql`), never the live data dir. `backup/run` now guards on a + fresh snapshot landing for the host (a no-label repo would otherwise be a silent no-op). diff --git a/actions/backup/run.py b/actions/backup/run.py index 420624b..f2223e6 100644 --- a/actions/backup/run.py +++ b/actions/backup/run.py @@ -1,17 +1,23 @@ # actions/backup/run.py -"""Port of sync/tiers restic-backup (SP4: behavior-identical). Runs the restic core -service one-shot: init-if-needed -> backup tiers -> forget --prune -> Telegram.""" +"""Label-driven restic backup via resticontainer (github.com/deytenit/resticontainer). +Runs the resticontainer core service one-shot: init-if-needed -> backup (discovers +`restic.*`-labelled services, runs their pre-hooks/stops, snapshots the union of the +resolved host paths in a single restic run) -> forget --prune -> Telegram. Every +non-`backup` subcommand passes straight through to restic, so init/snapshots/forget +behave exactly as before.""" +import json import os from engine.descriptor import Meta, Arg -METADATA = Meta(summary="Run restic backup of the backed-up tiers (init-if-needed, backup, forget --prune).", +METADATA = Meta(summary="Run the resticontainer label-driven backup (init-if-needed, backup, forget --prune).", args=[Arg("telegram_bot_url", "Telegram Bot URL for notifications")]) TITLE = "Restic Host Backup" def run(ctx, args): log, node, url = ctx.log, ctx.node.name, args.telegram_bot_url core = ctx.paths.core - tag = f"com-ermnvldmr-root-{node}" # KEPT unchanged for backup continuity (SP4/Decision D) + host = f"{node}.apex.ermnvldmr.com" # restic --host: the per-node public name (matches the bucket) + tag = "biweekly" # restic --tag: snapshot cadence (this cron fires on the 5th & 20th) if not os.path.isfile(os.path.join(core, "docker-compose.yml")): log.error(f"Core compose not found in {core}"); raise SystemExit(1) env = dict(os.environ, COMPOSE_PROFILES="manual") @@ -21,9 +27,20 @@ def run(ctx, args): if os.path.isfile(os.path.join(core, ef)): envfiles += ["--env-file", ef] - def restic(*cmd, check=True): + def restic(*cmd, check=True, capture=False): return ctx.sys.run(["docker", "compose", *envfiles, "--profile", "manual", - "run", "--rm", "restic", *cmd], cwd=core, env=env, check=check) + "run", "--rm", "-T", "resticontainer", *cmd], + cwd=core, env=env, check=check, capture=capture) + + def host_snapshot_ids(): + """This host's snapshot IDs, or None if it can't be determined (never fail on this).""" + cp = restic("snapshots", "--host", host, "--json", check=False, capture=True) + if cp.returncode != 0: + return None + try: + return {s.get("id") for s in json.loads(cp.stdout or "[]")} + except (ValueError, TypeError): + return None try: log.info("Checking restic repository availability...") @@ -31,10 +48,20 @@ def run(ctx, args): log.info("Restic repository reachable.") elif restic("init", check=False).returncode != 0: ctx.notify.error(TITLE, url, "restic init failed", node); raise SystemExit(1) - log.info("Backing up /data/@tier1 and /data/@tier2 ...") - if restic("backup", "/data/@tier1", "/data/@tier2", "--host", node, "--tag", tag, - "--compression", "auto", check=False).returncode != 0: + before = host_snapshot_ids() + log.info("Backing up `restic.*`-labelled services (resticontainer discovers paths)...") + if restic("backup", "--host", host, "--tag", tag, "--compression", "max", + check=False).returncode != 0: ctx.notify.error(TITLE, url, "restic backup failed", node); raise SystemExit(1) + # Guard: the label-driven backup is a silent no-op if no service carries the + # `restic.enable` label (resticontainer finds zero paths and skips restic). + # Prove a fresh snapshot for this host actually landed before we prune/notify. + after = host_snapshot_ids() + if before is not None and after is not None and not (after - before): + log.error("No new snapshot produced — refusing to report success.") + ctx.notify.error(TITLE, url, + "backup produced no new snapshot (no restic.* labels discovered?)", node) + raise SystemExit(1) log.info("Applying retention (forget --prune)...") if restic("forget", "--prune", "--keep-daily", "7", "--keep-weekly", "4", "--keep-monthly", "12", check=False).returncode != 0: diff --git a/compositions/apex/docker-compose.yml b/compositions/apex/docker-compose.yml index 2bd96a2..d35e9ff 100644 --- a/compositions/apex/docker-compose.yml +++ b/compositions/apex/docker-compose.yml @@ -27,30 +27,39 @@ services: - no-new-privileges:true # --- backups (host network, manual profile, one-shot) --- - restic: - image: restic/restic:0.18.1 - container_name: apex-restic + # resticontainer (github.com/deytenit/resticontainer): a restic wrapper that + # discovers per-service backup intent from `restic.*` compose labels + # (restic.enable / restic.backup.paths / restic.backup.stop / restic.hooks.*). + # On `backup` it queries the docker daemon, runs each service's pre-hook (e.g. + # pg_dump), stops any container flagged restic.backup.stop, resolves the labeled + # container mounts to host paths (via /hostfs + mount source), snapshots the union + # in a single restic run, then restarts + runs post-hooks. Every other subcommand + # (init/snapshots/forget/…) passes straight through to restic. Needs the docker + # socket (list/inspect/exec/stop/start) and the host root read-only (to read the + # data) — a privileged manual one-shot, same trust surface as the old restic core. + resticontainer: + image: ghcr.io/deytenit/resticontainer:0.19.1 + container_name: apex-resticontainer profiles: [manual] volumes: - ${APEX_TIER3}/restic/cache:/var/restic-cache - # FIX (Task 1.11): mount the backed-up tier roots read-only so `restic backup - # /data/@tier1 /data/@tier2` actually snapshots data (was missing today). - - ${APEX_TIER_ROOT1}:/data/@tier1:ro - - ${APEX_TIER_ROOT2}:/data/@tier2:ro + - /var/run/docker.sock:/var/run/docker.sock # query + exec/stop/start + - /:/hostfs:ro # read the data off the host environment: <<: *shared-environment + RESTIC_HOSTFS: /hostfs # prefix for resolved mount sources RESTIC_REPOSITORY: "${APEX_RESTIC_REPOSITORY?error}" RESTIC_PASSWORD: "${APEX_RESTIC_PASSWORD?error}" AWS_ACCESS_KEY_ID: "${APEX_RESTIC_AWS_ACCESS_KEY_ID?error}" AWS_SECRET_ACCESS_KEY: "${APEX_RESTIC_AWS_SECRET_ACCESS_KEY?error}" AWS_REGION: "ru-central1" AWS_ENDPOINT_URL: "https://storage.yandexcloud.net/" - # per-node override (icarus historically ran "max" for manual invocations; - # scheduled backups always force --compression auto on the CLI) - RESTIC_COMPRESSION: "${APEX_RESTIC_COMPRESSION:-auto}" + # max compression is fleet policy (backup/run also forces --compression max on + # the CLI; needs repo format v2, which a fresh restic init creates). Per-node + # override via APEX_RESTIC_COMPRESSION only if ever needed. + RESTIC_COMPRESSION: "${APEX_RESTIC_COMPRESSION:-max}" RESTIC_CACHE_DIR: /var/restic-cache network_mode: "host" - entrypoint: ["restic"] # --- docker socket proxy --- socket-proxy: