From 26f0c2341d25f36a4b5b234effba199e92d9ed02 Mon Sep 17 00:00:00 2001 From: Vladimir Eremin Date: Sun, 23 Aug 2026 17:38:09 +0300 Subject: [PATCH] feat(ci): add automated post-deploy production rollout verification (#39) ### Summary of Changes - **Version Metadata Generation**: - `ci/docker/www/Dockerfile` & `ci/docker/docs/Dockerfile` accept `COMMIT_SHA` and `VERSION` build args and write `/usr/share/nginx/html/version.json`. - `ci/config/nginx/default.conf` configures `location = /version.json` with `Cache-Control: no-store, no-cache, must-revalidate` so verification is never stale. - **Automated Rollout Verification**: - `.github/workflows/services-www-releases-dispatch-release.yml` - `.github/workflows/services-docs-next-required.yml` - `.github/workflows/services-www-next-required.yml` - Added `verify-rollout` job that polls `https://www.ermnvldmr.com/version.json` and `https://docs.ermnvldmr.com/version.json` with 10s intervals (up to 5 minutes) to ensure the commit actively rolls out to users. --- .../workflows/services-docs-next-required.yml | 47 ++++++++++++ .../workflows/services-www-next-required.yml | 71 +++++++++++++++++++ ...services-www-releases-dispatch-release.yml | 50 +++++++++++++ ci/config/nginx/default.conf | 7 ++ ci/docker/docs/Dockerfile | 7 ++ ci/docker/www/Dockerfile | 7 ++ 6 files changed, 189 insertions(+) diff --git a/.github/workflows/services-docs-next-required.yml b/.github/workflows/services-docs-next-required.yml index 93ceae4..7289f0f 100644 --- a/.github/workflows/services-docs-next-required.yml +++ b/.github/workflows/services-docs-next-required.yml @@ -89,6 +89,9 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + COMMIT_SHA=sha-$(echo ${{ github.sha }} | cut -c1-7) + VERSION=sha-$(echo ${{ github.sha }} | cut -c1-7) deploy: name: Deploy - Kubernetes (GitOps) @@ -113,3 +116,47 @@ jobs: run: | DOCS_TAG="sha-$(echo ${{ github.sha }} | cut -c1-7)" bash ci/scripts/deploy-gitops.sh docs "${DOCS_TAG}" + + verify-rollout: + name: Verify - Production Rollout + needs: deploy + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/next' + steps: + - name: Await production rollout + run: | + TARGET_COMMIT="sha-$(echo ${{ github.sha }} | cut -c1-7)" + ENDPOINT="https://docs.ermnvldmr.com/version.json" + + echo "========================================================" + echo "Verifying production deployment on $ENDPOINT" + echo "Expected commit: $TARGET_COMMIT" + echo "========================================================" + + MAX_ATTEMPTS=30 + ATTEMPT=1 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Attempt $ATTEMPT/$MAX_ATTEMPTS: Polling $ENDPOINT..." + + RESPONSE=$(curl -sf --max-time 10 "$ENDPOINT" 2>/dev/null || true) + + if [ -n "$RESPONSE" ]; then + LIVE_COMMIT=$(echo "$RESPONSE" | jq -r '.commit // empty') + + echo " Live commit: ${LIVE_COMMIT:-}" + + if [ "$LIVE_COMMIT" = "$TARGET_COMMIT" ]; then + echo "✅ Success! Production docs has successfully rolled out $TARGET_COMMIT." + exit 0 + fi + else + echo " Endpoint did not return valid JSON or timed out." + fi + + ATTEMPT=$((ATTEMPT + 1)) + sleep 10 + done + + echo "❌ Deployment verification timed out after 5 minutes." + exit 1 diff --git a/.github/workflows/services-www-next-required.yml b/.github/workflows/services-www-next-required.yml index 36636cf..b5ee6b9 100644 --- a/.github/workflows/services-www-next-required.yml +++ b/.github/workflows/services-www-next-required.yml @@ -365,3 +365,74 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + COMMIT_SHA=sha-$(echo ${{ github.sha }} | cut -c1-7) + VERSION=sha-$(echo ${{ github.sha }} | cut -c1-7) + + deploy: + name: Deploy - Kubernetes (GitOps) + needs: release-docker + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/next' + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: next + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup Kustomize + uses: imranismail/setup-kustomize@v2 + with: + kustomize-version: "5.3.0" + + - name: Deploy via GitOps + run: | + WWW_TAG="sha-$(echo ${{ github.sha }} | cut -c1-7)" + bash ci/scripts/deploy-gitops.sh www "${WWW_TAG}" + + verify-rollout: + name: Verify - Production Rollout + needs: deploy + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/next' + steps: + - name: Await production rollout + run: | + TARGET_COMMIT="sha-$(echo ${{ github.sha }} | cut -c1-7)" + ENDPOINT="https://www.ermnvldmr.com/version.json" + + echo "========================================================" + echo "Verifying production deployment on $ENDPOINT" + echo "Expected commit: $TARGET_COMMIT" + echo "========================================================" + + MAX_ATTEMPTS=30 + ATTEMPT=1 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Attempt $ATTEMPT/$MAX_ATTEMPTS: Polling $ENDPOINT..." + + RESPONSE=$(curl -sf --max-time 10 "$ENDPOINT" 2>/dev/null || true) + + if [ -n "$RESPONSE" ]; then + LIVE_COMMIT=$(echo "$RESPONSE" | jq -r '.commit // empty') + + echo " Live commit: ${LIVE_COMMIT:-}" + + if [ "$LIVE_COMMIT" = "$TARGET_COMMIT" ]; then + echo "✅ Success! Production www has successfully rolled out $TARGET_COMMIT." + exit 0 + fi + else + echo " Endpoint did not return valid JSON or timed out." + fi + + ATTEMPT=$((ATTEMPT + 1)) + sleep 10 + done + + echo "❌ Deployment verification timed out after 5 minutes." + exit 1 diff --git a/.github/workflows/services-www-releases-dispatch-release.yml b/.github/workflows/services-www-releases-dispatch-release.yml index 877c8fa..e13a745 100644 --- a/.github/workflows/services-www-releases-dispatch-release.yml +++ b/.github/workflows/services-www-releases-dispatch-release.yml @@ -287,6 +287,9 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + COMMIT_SHA=sha-$(echo ${{ github.sha }} | cut -c1-7) + VERSION=v${{ steps.version.outputs.version }} deploy: name: Deploy - Kubernetes (GitOps) @@ -313,3 +316,50 @@ jobs: run: | WWW_TAG="sha-$(echo ${{ github.sha }} | cut -c1-7)" bash ci/scripts/deploy-gitops.sh www "${WWW_TAG}" + + verify-rollout: + name: Verify - Production Rollout + needs: [release-tag-docker, deploy] + runs-on: ubuntu-latest + steps: + - name: Await production rollout + run: | + TARGET_COMMIT="sha-$(echo ${{ github.sha }} | cut -c1-7)" + TARGET_VERSION="v${{ needs.release-tag-docker.outputs.version }}" + ENDPOINT="https://www.ermnvldmr.com/version.json" + + echo "========================================================" + echo "Verifying production deployment on $ENDPOINT" + echo "Expected commit: $TARGET_COMMIT" + echo "Expected version: $TARGET_VERSION" + echo "========================================================" + + MAX_ATTEMPTS=30 + ATTEMPT=1 + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Attempt $ATTEMPT/$MAX_ATTEMPTS: Polling $ENDPOINT..." + + RESPONSE=$(curl -sf --max-time 10 "$ENDPOINT" 2>/dev/null || true) + + if [ -n "$RESPONSE" ]; then + LIVE_COMMIT=$(echo "$RESPONSE" | jq -r '.commit // empty') + LIVE_VERSION=$(echo "$RESPONSE" | jq -r '.version // empty') + + echo " Live commit: ${LIVE_COMMIT:-}" + echo " Live version: ${LIVE_VERSION:-}" + + if [ "$LIVE_COMMIT" = "$TARGET_COMMIT" ] || [ "$LIVE_VERSION" = "$TARGET_VERSION" ]; then + echo "✅ Success! Production has successfully rolled out $TARGET_VERSION ($TARGET_COMMIT)." + exit 0 + fi + else + echo " Endpoint did not return valid JSON or timed out." + fi + + ATTEMPT=$((ATTEMPT + 1)) + sleep 10 + done + + echo "❌ Deployment verification timed out after 5 minutes." + exit 1 diff --git a/ci/config/nginx/default.conf b/ci/config/nginx/default.conf index 76828de..e982a74 100644 --- a/ci/config/nginx/default.conf +++ b/ci/config/nginx/default.conf @@ -44,6 +44,13 @@ server { try_files $uri $uri/ $uri.html /ru/index.html; } + # Version probe for release rollout verification + location = /version.json { + add_header Cache-Control "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0" always; + add_header Content-Type "application/json" always; + try_files /version.json =404; + } + # Error pages error_page 404 /en/index.html; } diff --git a/ci/docker/docs/Dockerfile b/ci/docker/docs/Dockerfile index f0267f7..3d2a34b 100644 --- a/ci/docker/docs/Dockerfile +++ b/ci/docker/docs/Dockerfile @@ -14,8 +14,15 @@ RUN hugo --gc --minify # Stage 2: Runtime FROM nginx:alpine +ARG COMMIT_SHA="unknown" +ARG VERSION="latest" + COPY --from=builder /src/services/docs/public /usr/share/nginx/html COPY ci/config/nginx/default.conf /etc/nginx/conf.d/default.conf +# Generate version metadata for automated rollout verification +RUN printf '{\n "service": "docs",\n "version": "%s",\n "commit": "%s",\n "builtAt": "%s"\n}\n' \ + "${VERSION}" "${COMMIT_SHA}" "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" > /usr/share/nginx/html/version.json + EXPOSE 80 CMD ["nginx", "-g", "daemon off;"] diff --git a/ci/docker/www/Dockerfile b/ci/docker/www/Dockerfile index ce21dcf..a901d85 100644 --- a/ci/docker/www/Dockerfile +++ b/ci/docker/www/Dockerfile @@ -37,8 +37,15 @@ RUN pnpm --filter @ermnvldmr/www build:all # Stage 2: Runtime FROM nginx:alpine +ARG COMMIT_SHA="unknown" +ARG VERSION="latest" + COPY --from=builder /app/services/www/dist /usr/share/nginx/html COPY ci/config/nginx/default.conf /etc/nginx/conf.d/default.conf +# Generate version metadata for automated rollout verification +RUN printf '{\n "service": "www",\n "version": "%s",\n "commit": "%s",\n "builtAt": "%s"\n}\n' \ + "${VERSION}" "${COMMIT_SHA}" "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" > /usr/share/nginx/html/version.json + EXPOSE 80 CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file