From 984dd04fe1b69579dcd29b8f600b73fa82149764 Mon Sep 17 00:00:00 2001 From: Vladimir Eremin Date: Sun, 8 Nov 2026 01:21:48 +0300 Subject: [PATCH] ci: streamline docs workflows, migrate scripts to ci/scripts and update www image --- .../workflows/services-docs-next-required.yml | 85 ++++++------- .../workflows/services-docs-pr-required.yml | 21 ++-- ci/k8s/apps/www/kustomization.yaml | 2 +- ci/scripts/await-rollout.sh | 63 ++++++++++ ci/scripts/cache.py | 87 ++++++++++++++ ci/scripts/deploy-gitops.sh | 6 +- ci/scripts/docker.py | 64 ++++++++++ ci/scripts/git_write.py | 27 +++++ ci/scripts/provenance.py | 113 ++++++++++++++++++ ci/scripts/setup-env.sh | 35 ++++++ 10 files changed, 440 insertions(+), 63 deletions(-) create mode 100755 ci/scripts/await-rollout.sh create mode 100644 ci/scripts/cache.py create mode 100644 ci/scripts/docker.py create mode 100644 ci/scripts/git_write.py create mode 100644 ci/scripts/provenance.py create mode 100755 ci/scripts/setup-env.sh diff --git a/.gitea/workflows/services-docs-next-required.yml b/.gitea/workflows/services-docs-next-required.yml index 95c2829..a57f060 100644 --- a/.gitea/workflows/services-docs-next-required.yml +++ b/.gitea/workflows/services-docs-next-required.yml @@ -9,45 +9,43 @@ name: services/docs - next/required - ci/docker/docs/** - ci/config/nginx/** - ci/scripts/** - - .gitea/scripts/** - - .gitea/actions/** + workflow_dispatch: + concurrency: group: ci-docs-${{ github.ref_name }} cancel-in-progress: true + permissions: contents: read actions: read + jobs: setup-hugo: name: Setup - Hugo runs-on: apex-debian13 steps: - name: Checkout repository - run: "set -euo pipefail\ncase \"$CHECKOUT_REF\" in -*) exit 1;; esac\nGITHUB_SERVER_URL=${GITHUB_SERVER_URL%/}\ncase \"$GITHUB_SERVER_URL\" in https://git.ermnvldmr.com) ;; *) exit 1;; esac\nAUTH=$(printf 'deytenit:%s' \"$CHECKOUT_TOKEN\" | base64 -w0)\nHEADER=\"http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION: basic $AUTH\"\ngit init \"$GITHUB_WORKSPACE\"\ncd \"$GITHUB_WORKSPACE\"\ngit remote add origin \"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git\"\ngit -c \"$HEADER\" fetch --tags origin '+refs/heads/*:refs/remotes/origin/*'\ngit -c \"$HEADER\" fetch origin \"$CHECKOUT_REF\"\ngit checkout --detach FETCH_HEAD\ntest \"$(git rev-parse HEAD)\" = \"$GITHUB_SHA\"\nif [ \"$CHECKOUT_SUBMODULES\" = recursive ]; then\n git submodule update --init --recursive\nfi\nif [ \"$PERSIST_CREDENTIALS\" = true ]; then\n git config \"http.${GITHUB_SERVER_URL}/.extraheader\" \"AUTHORIZATION: basic $AUTH\"\nfi\n" - env: - CHECKOUT_REF: ${{ github.sha }} - CHECKOUT_SUBMODULES: recursive - CHECKOUT_TOKEN: ${{ github.token }} - PERSIST_CREDENTIALS: 'false' + uses: actions/checkout@v4 + with: + submodules: recursive - name: Setup Hugo run: hugo version | grep extended + build-hugo: name: Build - Hugo Site needs: setup-hugo runs-on: apex-debian13 steps: - name: Checkout repository - run: "set -euo pipefail\ncase \"$CHECKOUT_REF\" in -*) exit 1;; esac\nGITHUB_SERVER_URL=${GITHUB_SERVER_URL%/}\ncase \"$GITHUB_SERVER_URL\" in https://git.ermnvldmr.com) ;; *) exit 1;; esac\nAUTH=$(printf 'deytenit:%s' \"$CHECKOUT_TOKEN\" | base64 -w0)\nHEADER=\"http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION: basic $AUTH\"\ngit init \"$GITHUB_WORKSPACE\"\ncd \"$GITHUB_WORKSPACE\"\ngit remote add origin \"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git\"\ngit -c \"$HEADER\" fetch --tags origin '+refs/heads/*:refs/remotes/origin/*'\ngit -c \"$HEADER\" fetch origin \"$CHECKOUT_REF\"\ngit checkout --detach FETCH_HEAD\ntest \"$(git rev-parse HEAD)\" = \"$GITHUB_SHA\"\nif [ \"$CHECKOUT_SUBMODULES\" = recursive ]; then\n git submodule update --init --recursive\nfi\nif [ \"$PERSIST_CREDENTIALS\" = true ]; then\n git config \"http.${GITHUB_SERVER_URL}/.extraheader\" \"AUTHORIZATION: basic $AUTH\"\nfi\n" - env: - CHECKOUT_REF: ${{ github.sha }} - CHECKOUT_SUBMODULES: recursive - CHECKOUT_TOKEN: ${{ github.token }} - PERSIST_CREDENTIALS: 'false' + uses: actions/checkout@v4 + with: + submodules: recursive - name: Setup Hugo run: hugo version | grep extended - name: Build Hugo site working-directory: services/docs run: hugo --gc --minify + release-docker: name: Release - Docker Image needs: build-hugo @@ -57,48 +55,43 @@ jobs: packages: write steps: - name: Checkout repository - run: "set -euo pipefail\ncase \"$CHECKOUT_REF\" in -*) exit 1;; esac\nGITHUB_SERVER_URL=${GITHUB_SERVER_URL%/}\ncase \"$GITHUB_SERVER_URL\" in https://git.ermnvldmr.com) ;; *) exit 1;; esac\nAUTH=$(printf 'deytenit:%s' \"$CHECKOUT_TOKEN\" | base64 -w0)\nHEADER=\"http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION: basic $AUTH\"\ngit init \"$GITHUB_WORKSPACE\"\ncd \"$GITHUB_WORKSPACE\"\ngit remote add origin \"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git\"\ngit -c \"$HEADER\" fetch --tags origin '+refs/heads/*:refs/remotes/origin/*'\ngit -c \"$HEADER\" fetch origin \"$CHECKOUT_REF\"\ngit checkout --detach FETCH_HEAD\ntest \"$(git rev-parse HEAD)\" = \"$GITHUB_SHA\"\nif [ \"$CHECKOUT_SUBMODULES\" = recursive ]; then\n git submodule update --init --recursive\nfi\nif [ \"$PERSIST_CREDENTIALS\" = true ]; then\n git config \"http.${GITHUB_SERVER_URL}/.extraheader\" \"AUTHORIZATION: basic $AUTH\"\nfi\n" - env: - CHECKOUT_REF: ${{ github.sha }} - CHECKOUT_SUBMODULES: recursive - CHECKOUT_TOKEN: ${{ github.token }} - PERSIST_CREDENTIALS: 'false' - - name: Log in to GitHub Container Registry - run: python3 .gitea/scripts/docker.py login + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + - name: Log in to Gitea Container Registry + run: python3 ci/scripts/docker.py login env: REGISTRY: git.ermnvldmr.com REGISTRY_USER: ${{ github.repository_owner }} REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - name: Extract Docker metadata id: meta - run: python3 .gitea/scripts/docker.py metadata + run: python3 ci/scripts/docker.py metadata env: IMAGES: git.ermnvldmr.com/${{ github.repository }}/docs - TAG_RULES: 'type=raw,value=latest - + TAG_RULES: | + type=raw,value=latest type=sha,format=short - - ' SOURCE_SHA: ${{ github.sha }} - name: Compute short SHA id: vars run: echo "short_sha=sha-$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT - name: Build and push Docker image - run: python3 .gitea/scripts/docker.py build + run: python3 ci/scripts/docker.py build env: TAGS: ${{ steps.meta.outputs.tags }} LABELS: ${{ steps.meta.outputs.labels }} - BUILD_ARGS: 'COMMIT_SHA=${{ steps.vars.outputs.short_sha }} - + BUILD_ARGS: | + COMMIT_SHA=${{ steps.vars.outputs.short_sha }} VERSION=${{ steps.vars.outputs.short_sha }} - - ' DOCKERFILE: ci/docker/docs/Dockerfile PLATFORMS: '' PUSH: 'true' CONTEXT: . SOURCE_SHA: ${{ github.sha }} RUN_ID: ${{ github.run_id }} + deploy: name: Deploy - Kubernetes (GitOps) needs: release-docker @@ -108,31 +101,29 @@ jobs: contents: write steps: - name: Checkout repository - run: "set -euo pipefail\ncase \"$CHECKOUT_REF\" in -*) exit 1;; esac\nGITHUB_SERVER_URL=${GITHUB_SERVER_URL%/}\ncase \"$GITHUB_SERVER_URL\" in https://git.ermnvldmr.com) ;; *) exit 1;; esac\nAUTH=$(printf 'deytenit:%s' \"$CHECKOUT_TOKEN\" | base64 -w0)\nHEADER=\"http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION: basic $AUTH\"\ngit init \"$GITHUB_WORKSPACE\"\ncd \"$GITHUB_WORKSPACE\"\ngit remote add origin \"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git\"\ngit -c \"$HEADER\" fetch --tags origin '+refs/heads/*:refs/remotes/origin/*'\ngit -c \"$HEADER\" fetch origin \"$CHECKOUT_REF\"\ngit checkout --detach FETCH_HEAD\ntest \"$(git rev-parse HEAD)\" = \"$GITHUB_SHA\"\nif [ \"$CHECKOUT_SUBMODULES\" = recursive ]; then\n key_file=$(mktemp \"${RUNNER_TEMP:-/tmp}/hextra-key.XXXXXX\")\n trap 'rm -f \"$key_file\"' EXIT\n umask 077\n printf '%s\\n' \"$HEXTRA_READ_KEY\" > \"$key_file\"\n chmod 600 \"$key_file\"\n export GIT_SSH_COMMAND=\"ssh -T -i $key_file -o IdentitiesOnly=yes\ - \ -o StrictHostKeyChecking=yes -o HostKeyAlgorithms=ssh-ed25519 -o UserKnownHostsFile=$GITHUB_WORKSPACE/.gitea/known_hosts\"\n git submodule update --init --recursive\nfi\nif [ \"$PERSIST_CREDENTIALS\" = true ]; then\n git config \"http.${GITHUB_SERVER_URL}/.extraheader\" \"AUTHORIZATION: basic $AUTH\"\nfi\n" - env: - CHECKOUT_REF: ${{ github.sha }} - CHECKOUT_SUBMODULES: 'false' - CHECKOUT_TOKEN: ${{ github.token }} - PERSIST_CREDENTIALS: 'false' - HEXTRA_READ_KEY: '' + uses: actions/checkout@v4 + with: + ref: next - name: Setup Kustomize run: test "$(kustomize version)" = v5.3.0 - name: Deploy via GitOps - run: 'DOCS_TAG="sha-$(echo ${{ github.sha }} | cut -c1-7)" - + run: | + DOCS_TAG="sha-$(echo ${{ github.sha }} | cut -c1-7)" bash ci/scripts/deploy-gitops.sh docs "${DOCS_TAG}" - - ' env: GIT_WRITE_KEY: ${{ secrets.DOCS_DEPLOY_KEY }} + verify-rollout: name: Verify - Production Rollout needs: deploy runs-on: apex-debian13 if: github.event_name == 'push' && github.ref == 'refs/heads/next' steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: next - name: Await production rollout - run: "TARGET_COMMIT=\"sha-$(echo ${{ github.sha }} | cut -c1-7)\"\nENDPOINT=\"https://docs.ermnvldmr.com/version.json\"\n\necho \"========================================================\"\necho \"Verifying production deployment on $ENDPOINT\"\necho \"Expected commit: $TARGET_COMMIT\"\necho \"========================================================\"\n\nMAX_ATTEMPTS=45\nATTEMPT=1\nCF_BLOCKED_COUNT=0\n\nwhile [ $ATTEMPT -le $MAX_ATTEMPTS ]; do\n echo \"Attempt $ATTEMPT/$MAX_ATTEMPTS: Polling $ENDPOINT...\"\n\n RESP_FILE=$(mktemp)\n HTTP_CODE=$(curl -sL --max-time 10 \\\n -H \"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36\" \\\n -H \"Accept: application/json\" \\\n -o \"$RESP_FILE\" \\\n -w \"%{http_code}\" \\\n \"$ENDPOINT\" || echo \"000\")\n\n if [ \"$HTTP_CODE\" = \"200\" ]; then\n LIVE_COMMIT=$(jq -r '.commit // empty' \"$RESP_FILE\" 2>/dev/null || true)\n echo \" Live commit:\ - \ ${LIVE_COMMIT:-}\"\n\n if [ \"$LIVE_COMMIT\" = \"$TARGET_COMMIT\" ]; then\n echo \"\u2705 Success! Production docs has successfully rolled out $TARGET_COMMIT.\"\n rm -f \"$RESP_FILE\"\n exit 0\n fi\n elif [ \"$HTTP_CODE\" = \"403\" ] || [ \"$HTTP_CODE\" = \"503\" ]; then\n echo \" HTTP $HTTP_CODE: Cloudflare WAF/challenge encountered from runner IP.\"\n CF_BLOCKED_COUNT=$((CF_BLOCKED_COUNT + 1))\n else\n echo \" HTTP $HTTP_CODE: Endpoint returned unexpected status code or timed out.\"\n fi\n\n rm -f \"$RESP_FILE\"\n\n if [ $CF_BLOCKED_COUNT -ge 5 ]; then\n echo \"\u26A0\uFE0F Cloudflare WAF consistently blocked external probe requests from this runner IP (HTTP $HTTP_CODE).\"\n echo \"The GitOps release commit was already pushed to the repository; FluxCD will reconcile the cluster.\"\n echo \"Skipping further external HTTP verification to avoid false negative.\"\n exit 0\n fi\n\n ATTEMPT=$((ATTEMPT + 1))\n sleep 10\ndone\n\ - \necho \"\u274C Deployment verification timed out after 7.5 minutes.\"\nexit 1\n" + run: | + DOCS_TAG="sha-$(echo ${{ github.sha }} | cut -c1-7)" + bash ci/scripts/await-rollout.sh docs "${DOCS_TAG}" diff --git a/.gitea/workflows/services-docs-pr-required.yml b/.gitea/workflows/services-docs-pr-required.yml index 497124b..64337b6 100644 --- a/.gitea/workflows/services-docs-pr-required.yml +++ b/.gitea/workflows/services-docs-pr-required.yml @@ -6,8 +6,7 @@ name: services/docs - pr/required paths: - services/docs/** - .gitea/workflows/services-docs-pr-required.yml - - .gitea/scripts/** - - .gitea/actions/** + - ci/scripts/** concurrency: group: ci-pr-docs-${{ github.head_ref }} cancel-in-progress: true @@ -20,12 +19,9 @@ jobs: runs-on: apex-debian13 steps: - name: Checkout repository - run: "set -euo pipefail\ncase \"$CHECKOUT_REF\" in -*) exit 1;; esac\nGITHUB_SERVER_URL=${GITHUB_SERVER_URL%/}\ncase \"$GITHUB_SERVER_URL\" in https://git.ermnvldmr.com) ;; *) exit 1;; esac\nAUTH=$(printf 'deytenit:%s' \"$CHECKOUT_TOKEN\" | base64 -w0)\nHEADER=\"http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION: basic $AUTH\"\ngit init \"$GITHUB_WORKSPACE\"\ncd \"$GITHUB_WORKSPACE\"\ngit remote add origin \"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git\"\ngit -c \"$HEADER\" fetch --tags origin '+refs/heads/*:refs/remotes/origin/*'\ngit -c \"$HEADER\" fetch origin \"$CHECKOUT_REF\"\ngit checkout --detach FETCH_HEAD\ntest \"$(git rev-parse HEAD)\" = \"$GITHUB_SHA\"\nif [ \"$CHECKOUT_SUBMODULES\" = recursive ]; then\n git submodule update --init --recursive\nfi\nif [ \"$PERSIST_CREDENTIALS\" = true ]; then\n git config \"http.${GITHUB_SERVER_URL}/.extraheader\" \"AUTHORIZATION: basic $AUTH\"\nfi\n" - env: - CHECKOUT_REF: ${{ github.sha }} - CHECKOUT_SUBMODULES: recursive - CHECKOUT_TOKEN: ${{ github.token }} - PERSIST_CREDENTIALS: 'false' + uses: actions/checkout@v4 + with: + submodules: recursive - name: Setup Hugo run: hugo version | grep extended build-hugo: @@ -34,12 +30,9 @@ jobs: runs-on: apex-debian13 steps: - name: Checkout repository - run: "set -euo pipefail\ncase \"$CHECKOUT_REF\" in -*) exit 1;; esac\nGITHUB_SERVER_URL=${GITHUB_SERVER_URL%/}\ncase \"$GITHUB_SERVER_URL\" in https://git.ermnvldmr.com) ;; *) exit 1;; esac\nAUTH=$(printf 'deytenit:%s' \"$CHECKOUT_TOKEN\" | base64 -w0)\nHEADER=\"http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION: basic $AUTH\"\ngit init \"$GITHUB_WORKSPACE\"\ncd \"$GITHUB_WORKSPACE\"\ngit remote add origin \"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git\"\ngit -c \"$HEADER\" fetch --tags origin '+refs/heads/*:refs/remotes/origin/*'\ngit -c \"$HEADER\" fetch origin \"$CHECKOUT_REF\"\ngit checkout --detach FETCH_HEAD\ntest \"$(git rev-parse HEAD)\" = \"$GITHUB_SHA\"\nif [ \"$CHECKOUT_SUBMODULES\" = recursive ]; then\n git submodule update --init --recursive\nfi\nif [ \"$PERSIST_CREDENTIALS\" = true ]; then\n git config \"http.${GITHUB_SERVER_URL}/.extraheader\" \"AUTHORIZATION: basic $AUTH\"\nfi\n" - env: - CHECKOUT_REF: ${{ github.sha }} - CHECKOUT_SUBMODULES: recursive - CHECKOUT_TOKEN: ${{ github.token }} - PERSIST_CREDENTIALS: 'false' + uses: actions/checkout@v4 + with: + submodules: recursive - name: Setup Hugo run: hugo version | grep extended - name: Build Hugo site diff --git a/ci/k8s/apps/www/kustomization.yaml b/ci/k8s/apps/www/kustomization.yaml index 7e32964..11959d9 100644 --- a/ci/k8s/apps/www/kustomization.yaml +++ b/ci/k8s/apps/www/kustomization.yaml @@ -6,4 +6,4 @@ resources: - service.yaml images: - name: git.ermnvldmr.com/deytenit/ermnvldmr.com/www - newTag: sha-84772ff + newTag: sha-fea1928 diff --git a/ci/scripts/await-rollout.sh b/ci/scripts/await-rollout.sh new file mode 100755 index 0000000..31be554 --- /dev/null +++ b/ci/scripts/await-rollout.sh @@ -0,0 +1,63 @@ +#!/bin/bash +set -euo pipefail + +SERVICE="${1:-}" +TARGET_COMMIT="${2:-}" +ENDPOINT="${3:-https://${SERVICE}.ermnvldmr.com/version.json}" + +if [ -z "$SERVICE" ] || [ -z "$TARGET_COMMIT" ]; then + echo "Usage: $0 [endpoint_url]" >&2 + exit 1 +fi + +echo "========================================================" +echo "Verifying production deployment on $ENDPOINT" +echo "Expected commit: $TARGET_COMMIT" +echo "========================================================" + +MAX_ATTEMPTS=45 +ATTEMPT=1 +CF_BLOCKED_COUNT=0 + +while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Attempt $ATTEMPT/$MAX_ATTEMPTS: Polling $ENDPOINT..." + + RESP_FILE=$(mktemp) + HTTP_CODE=$(curl -sL --max-time 10 \ + -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" \ + -H "Accept: application/json" \ + -o "$RESP_FILE" \ + -w "%{http_code}" \ + "$ENDPOINT" || echo "000") + + if [ "$HTTP_CODE" = "200" ]; then + LIVE_COMMIT=$(jq -r '.commit // empty' "$RESP_FILE" 2>/dev/null || true) + echo " Live commit: ${LIVE_COMMIT:-}" + + if [ "$LIVE_COMMIT" = "$TARGET_COMMIT" ]; then + echo "✅ Success! Production $SERVICE has successfully rolled out $TARGET_COMMIT." + rm -f "$RESP_FILE" + exit 0 + fi + elif [ "$HTTP_CODE" = "403" ] || [ "$HTTP_CODE" = "503" ]; then + echo " HTTP $HTTP_CODE: Cloudflare WAF/challenge encountered from runner IP." + CF_BLOCKED_COUNT=$((CF_BLOCKED_COUNT + 1)) + else + echo " HTTP $HTTP_CODE: Endpoint returned unexpected status code or timed out." + fi + + rm -f "$RESP_FILE" + + if [ $CF_BLOCKED_COUNT -ge 5 ]; then + echo "⚠️ Cloudflare WAF consistently blocked external probe requests from this runner IP (HTTP $HTTP_CODE)." + echo "The GitOps release commit was already pushed to the repository; FluxCD will reconcile the cluster." + echo "Skipping further external HTTP verification to avoid false negative." + exit 0 + fi + + ATTEMPT=$((ATTEMPT + 1)) + sleep 10 +done + +echo "❌ Deployment verification timed out after 7.5 minutes." +exit 1 diff --git a/ci/scripts/cache.py b/ci/scripts/cache.py new file mode 100644 index 0000000..18c7048 --- /dev/null +++ b/ci/scripts/cache.py @@ -0,0 +1,87 @@ +import hashlib +import json +import os +import platform +import subprocess +import sys +import tarfile +import tempfile +import urllib.error +import urllib.parse +import urllib.request +import zipfile +from pathlib import Path + +SERVER='https://git.ermnvldmr.com' + + +def eligible(run, repository, current_run): + if run.get('repository',{}).get('full_name')!=repository: + return False + if str(run.get('id'))==current_run: + return True + return run.get('head_branch')=='next' and run.get('event') in ('push','workflow_dispatch') and run.get('status')=='completed' and run.get('conclusion')=='success' + + +def request(url): + if not url.startswith(SERVER+'/'): + raise ValueError('Unexpected cache artifact host') + class LocalRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if not newurl.startswith(SERVER+'/'): + raise ValueError('Unexpected cache storage host') + return super().redirect_request(req,fp,code,msg,headers,newurl) + req=urllib.request.Request(url,headers={'Authorization':'token '+os.environ['CACHE_TOKEN']}) + return urllib.request.build_opener(LocalRedirect).open(req,timeout=60) + + +def api(path): + with request(SERVER+'/api/v1/repos/'+os.environ['GITHUB_REPOSITORY']+path) as response: + return json.load(response) + + +def main(): + store=Path(subprocess.check_output(['pnpm','store','path','--silent'],text=True).strip()) + archive=Path('.ci/pnpm-store.tar') + if sys.argv[1]=='pack': + archive.parent.mkdir(exist_ok=True) + store.mkdir(parents=True,exist_ok=True) + with tarfile.open(archive,'w') as target:target.add(store,arcname='.') + return + versions=subprocess.check_output(['node','--version'],text=True).strip()+'-'+subprocess.check_output(['pnpm','--version'],text=True).strip() + key='pnpm-'+platform.system().lower()+'-'+platform.machine()+'-'+versions+'-'+hashlib.sha256(Path('pnpm-lock.yaml').read_bytes()).hexdigest() + with open(os.environ['GITHUB_OUTPUT'],'a') as output:output.write('key='+key+'\n') + try: + artifacts=api('/actions/artifacts?name='+urllib.parse.quote(key))['artifacts'] + artifacts.sort(key=lambda a:a['id'],reverse=True) + for artifact in artifacts: + if artifact.get('expired',True) or artifact.get('name')!=key: + continue + run_id=artifact.get('workflow_run',{}).get('id') + if not run_id: + continue + run=api('/actions/runs/'+str(run_id)) + if not eligible(run,os.environ['GITHUB_REPOSITORY'],os.environ['GITHUB_RUN_ID']): + continue + with tempfile.TemporaryDirectory(prefix='pnpm-artifact-cache-') as directory: + downloaded=Path(directory)/'cache.zip' + with request(SERVER+'/api/v1/repos/'+os.environ['GITHUB_REPOSITORY']+'/actions/artifacts/'+str(artifact['id'])+'/zip') as response,downloaded.open('wb') as target: + while chunk:=response.read(1048576):target.write(chunk) + with zipfile.ZipFile(downloaded) as zipped: + members=[name for name in zipped.namelist() if name.rsplit('/',1)[-1]=='pnpm-store.tar'] + if len(members)!=1: + raise ValueError('Invalid cache archive') + packed=Path(directory)/'store.tar' + with zipped.open(members[0]) as source,packed.open('wb') as target: + while chunk:=source.read(1048576):target.write(chunk) + store.mkdir(parents=True,exist_ok=True) + with tarfile.open(packed) as tar:tar.extractall(store,filter='data') + print('Restored repository-scoped pnpm cache artifact') + return + except (urllib.error.HTTPError,urllib.error.URLError,ValueError,tarfile.TarError,zipfile.BadZipFile): + print('Cache unavailable; dependencies will be fetched normally') + return + print('No eligible cache artifact; dependencies will be fetched normally') + + +if __name__=='__main__':main() diff --git a/ci/scripts/deploy-gitops.sh b/ci/scripts/deploy-gitops.sh index 4b371c0..fd5aed8 100755 --- a/ci/scripts/deploy-gitops.sh +++ b/ci/scripts/deploy-gitops.sh @@ -38,6 +38,10 @@ if git diff --staged --quiet; then echo "No manifest changes to commit for $SERVICE." else git commit -m "chore(release): deploy ${SERVICE} ${TAG} [skip ci]" - python3 .gitea/scripts/git_write.py HEAD:next + if [ -f "$ROOT_DIR/ci/scripts/git_write.py" ]; then + python3 "$ROOT_DIR/ci/scripts/git_write.py" HEAD:next + else + python3 "$ROOT_DIR/.gitea/scripts/git_write.py" HEAD:next + fi echo "Successfully deployed $SERVICE ($TAG) via GitOps." fi diff --git a/ci/scripts/docker.py b/ci/scripts/docker.py new file mode 100644 index 0000000..a4f310c --- /dev/null +++ b/ci/scripts/docker.py @@ -0,0 +1,64 @@ +import json +import os +import re +import subprocess +import sys +from pathlib import Path + + +def lines(value): + return [line.strip() for line in value.splitlines() if line.strip()] + + +def output(name, value): + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(f'{name}< 1024 * 1024: + raise ValueError('Unexpected provenance artifact size') + with zipfile.ZipFile(io.BytesIO(payload)) as archive: + members = [name for name in archive.namelist() if name.rsplit('/', 1)[-1] == filename] + if len(members) != 1: + raise ValueError('Missing or ambiguous provenance manifest') + if archive.getinfo(members[0]).file_size > 65536: + raise ValueError('Unexpected provenance document size') + return json.loads(archive.read(members[0])) + + +def main(): + sha, digest, run_id = os.environ['TESTED_SHA'], os.environ['TESTED_DIGEST'], os.environ['BUILD_RUN_ID'] + validate_input(sha, digest, run_id) + current = api('/actions/runs/' + os.environ['GITHUB_RUN_ID']) + if current.get('event') != 'workflow_dispatch' or current.get('head_branch') != 'next': + raise ValueError('Approval must be manually dispatched on protected next') + actor = current.get('trigger_actor') or current.get('actor') or {} + if actor.get('login') != 'deytenit': + raise ValueError('Only the repository owner may approve promotion') + if not api('/branches/next').get('protected'): + raise ValueError('The next branch must be protected before promotion') + validate_run(api('/actions/runs/' + run_id), sha) + validate_manifest(artifact_manifest(run_id), sha, digest, run_id) + if os.environ.get('RELEASE_RUN_ID'): + release_id = os.environ['RELEASE_RUN_ID'] + if not re.fullmatch(r'[1-9][0-9]*', release_id): + raise ValueError('Numeric release run ID required') + release = api('/actions/runs/' + release_id) + actor = release.get('trigger_actor') or release.get('actor') or {} + if release.get('conclusion') != 'success' or release.get('status') != 'completed' or release.get('event') != 'workflow_dispatch' or release.get('head_branch') != 'next' or release.get('path') != 'services-www-release-manual.yml@refs/heads/next' or actor.get('login') != 'deytenit': + raise ValueError('Successful independently approved release required') + manifest = artifact_manifest(release_id, 'www-release-provenance', 'www-release.json') + expected = {'sha': sha, 'digest': digest, 'run_id': run_id, 'repository': REPO, 'release_tag': os.environ['RELEASE_TAG']} + if manifest != expected: + raise ValueError('Release approval does not match requested artifact') + auth = base64.b64encode(('deytenit:' + os.environ['PROVENANCE_TOKEN']).encode()).decode() + subprocess.run(['git', '-c', 'http.https://git.ermnvldmr.com/.extraheader=AUTHORIZATION: basic ' + auth, 'fetch', 'origin', 'next'], check=True) + subprocess.run(['git', 'merge-base', '--is-ancestor', sha, 'origin/next'], check=True) + subprocess.run(['docker', 'pull', IMAGE + '@' + digest], check=True) + data = json.loads(subprocess.check_output(['docker', 'image', 'inspect', IMAGE + '@' + digest], text=True)) + validate_labels(data[0]['Config'].get('Labels') or {}, sha, run_id) + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write('short_sha=sha-' + sha[:7] + '\n') + + +if __name__ == '__main__': + main() diff --git a/ci/scripts/setup-env.sh b/ci/scripts/setup-env.sh new file mode 100755 index 0000000..02c3c07 --- /dev/null +++ b/ci/scripts/setup-env.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -euo pipefail + +PNPM_VERSION="${1:-9.12.3}" +NODE_VERSION="${2:-20}" + +# Configure pnpm +case "$PNPM_VERSION" in + 8|9|9.12.3) ;; + *) echo "Unsupported pnpm version: $PNPM_VERSION" >&2; exit 1 ;; +esac + +PNPM_BIN="/opt/ci/pnpm/$PNPM_VERSION/bin" +if [ -d "$PNPM_BIN" ]; then + if [ -n "${GITHUB_PATH:-}" ]; then + echo "$PNPM_BIN" >> "$GITHUB_PATH" + fi + export PATH="$PNPM_BIN:$PATH" +fi + +# Configure Node.js +case "$NODE_VERSION" in + 18*) major=18 ;; + 20*) major=20 ;; + 22*) major=22 ;; + *) echo "Unsupported node version: $NODE_VERSION" >&2; exit 1 ;; +esac + +NODE_BIN="/opt/ci/node/$major/bin" +if [ -d "$NODE_BIN" ]; then + if [ -n "${GITHUB_PATH:-}" ]; then + echo "$NODE_BIN" >> "$GITHUB_PATH" + fi + export PATH="$NODE_BIN:$PATH" +fi