feat(ci): add automated post-deploy production rollout verification (#39)

### Summary of Changes

- **Version Metadata Generation**:
- `ci/docker/www/Dockerfile` & `ci/docker/docs/Dockerfile` accept
`COMMIT_SHA` and `VERSION` build args and write
`/usr/share/nginx/html/version.json`.
- `ci/config/nginx/default.conf` configures `location = /version.json`
with `Cache-Control: no-store, no-cache, must-revalidate` so
verification is never stale.
- **Automated Rollout Verification**:
  - `.github/workflows/services-www-releases-dispatch-release.yml`
  - `.github/workflows/services-docs-next-required.yml`
  - `.github/workflows/services-www-next-required.yml`
- Added `verify-rollout` job that polls
`https://www.ermnvldmr.com/version.json` and
`https://docs.ermnvldmr.com/version.json` with 10s intervals (up to 5
minutes) to ensure the commit actively rolls out to users.

.github/workflows/services-docs-next-required.yml

@@ -89,6 +89,9 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            COMMIT_SHA=sha-$(echo ${{ github.sha }} | cut -c1-7)
+            VERSION=sha-$(echo ${{ github.sha }} | cut -c1-7)
 
   deploy:
     name: Deploy - Kubernetes (GitOps)
@@ -113,3 +116,47 @@ jobs:
         run: |
           DOCS_TAG="sha-$(echo ${{ github.sha }} | cut -c1-7)"
           bash ci/scripts/deploy-gitops.sh docs "${DOCS_TAG}"
+
+  verify-rollout:
+    name: Verify - Production Rollout
+    needs: deploy
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/next'
+    steps:
+      - name: Await production rollout
+        run: |
+          TARGET_COMMIT="sha-$(echo ${{ github.sha }} | cut -c1-7)"
+          ENDPOINT="https://docs.ermnvldmr.com/version.json"
+
+          echo "========================================================"
+          echo "Verifying production deployment on $ENDPOINT"
+          echo "Expected commit: $TARGET_COMMIT"
+          echo "========================================================"
+
+          MAX_ATTEMPTS=30
+          ATTEMPT=1
+
+          while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
+            echo "Attempt $ATTEMPT/$MAX_ATTEMPTS: Polling $ENDPOINT..."
+
+            RESPONSE=$(curl -sf --max-time 10 "$ENDPOINT" 2>/dev/null || true)
+
+            if [ -n "$RESPONSE" ]; then
+              LIVE_COMMIT=$(echo "$RESPONSE" | jq -r '.commit // empty')
+
+              echo "  Live commit: ${LIVE_COMMIT:-<none>}"
+
+              if [ "$LIVE_COMMIT" = "$TARGET_COMMIT" ]; then
+                echo "✅ Success! Production docs has successfully rolled out $TARGET_COMMIT."
+                exit 0
+              fi
+            else
+              echo "  Endpoint did not return valid JSON or timed out."
+            fi
+
+            ATTEMPT=$((ATTEMPT + 1))
+            sleep 10
+          done
+
+          echo "❌ Deployment verification timed out after 5 minutes."
+          exit 1

.github/workflows/services-www-next-required.yml

@@ -365,3 +365,74 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            COMMIT_SHA=sha-$(echo ${{ github.sha }} | cut -c1-7)
+            VERSION=sha-$(echo ${{ github.sha }} | cut -c1-7)
+
+  deploy:
+    name: Deploy - Kubernetes (GitOps)
+    needs: release-docker
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/next'
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: next
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Kustomize
+        uses: imranismail/setup-kustomize@v2
+        with:
+          kustomize-version: "5.3.0"
+
+      - name: Deploy via GitOps
+        run: |
+          WWW_TAG="sha-$(echo ${{ github.sha }} | cut -c1-7)"
+          bash ci/scripts/deploy-gitops.sh www "${WWW_TAG}"
+
+  verify-rollout:
+    name: Verify - Production Rollout
+    needs: deploy
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/next'
+    steps:
+      - name: Await production rollout
+        run: |
+          TARGET_COMMIT="sha-$(echo ${{ github.sha }} | cut -c1-7)"
+          ENDPOINT="https://www.ermnvldmr.com/version.json"
+
+          echo "========================================================"
+          echo "Verifying production deployment on $ENDPOINT"
+          echo "Expected commit: $TARGET_COMMIT"
+          echo "========================================================"
+
+          MAX_ATTEMPTS=30
+          ATTEMPT=1
+
+          while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
+            echo "Attempt $ATTEMPT/$MAX_ATTEMPTS: Polling $ENDPOINT..."
+
+            RESPONSE=$(curl -sf --max-time 10 "$ENDPOINT" 2>/dev/null || true)
+
+            if [ -n "$RESPONSE" ]; then
+              LIVE_COMMIT=$(echo "$RESPONSE" | jq -r '.commit // empty')
+
+              echo "  Live commit: ${LIVE_COMMIT:-<none>}"
+
+              if [ "$LIVE_COMMIT" = "$TARGET_COMMIT" ]; then
+                echo "✅ Success! Production www has successfully rolled out $TARGET_COMMIT."
+                exit 0
+              fi
+            else
+              echo "  Endpoint did not return valid JSON or timed out."
+            fi
+
+            ATTEMPT=$((ATTEMPT + 1))
+            sleep 10
+          done
+
+          echo "❌ Deployment verification timed out after 5 minutes."
+          exit 1

.github/workflows/services-www-releases-dispatch-release.yml

@@ -287,6 +287,9 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            COMMIT_SHA=sha-$(echo ${{ github.sha }} | cut -c1-7)
+            VERSION=v${{ steps.version.outputs.version }}
 
   deploy:
     name: Deploy - Kubernetes (GitOps)
@@ -313,3 +316,50 @@ jobs:
         run: |
           WWW_TAG="sha-$(echo ${{ github.sha }} | cut -c1-7)"
           bash ci/scripts/deploy-gitops.sh www "${WWW_TAG}"
+
+  verify-rollout:
+    name: Verify - Production Rollout
+    needs: [release-tag-docker, deploy]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Await production rollout
+        run: |
+          TARGET_COMMIT="sha-$(echo ${{ github.sha }} | cut -c1-7)"
+          TARGET_VERSION="v${{ needs.release-tag-docker.outputs.version }}"
+          ENDPOINT="https://www.ermnvldmr.com/version.json"
+
+          echo "========================================================"
+          echo "Verifying production deployment on $ENDPOINT"
+          echo "Expected commit:  $TARGET_COMMIT"
+          echo "Expected version: $TARGET_VERSION"
+          echo "========================================================"
+
+          MAX_ATTEMPTS=30
+          ATTEMPT=1
+
+          while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
+            echo "Attempt $ATTEMPT/$MAX_ATTEMPTS: Polling $ENDPOINT..."
+
+            RESPONSE=$(curl -sf --max-time 10 "$ENDPOINT" 2>/dev/null || true)
+
+            if [ -n "$RESPONSE" ]; then
+              LIVE_COMMIT=$(echo "$RESPONSE" | jq -r '.commit // empty')
+              LIVE_VERSION=$(echo "$RESPONSE" | jq -r '.version // empty')
+
+              echo "  Live commit:  ${LIVE_COMMIT:-<none>}"
+              echo "  Live version: ${LIVE_VERSION:-<none>}"
+
+              if [ "$LIVE_COMMIT" = "$TARGET_COMMIT" ] || [ "$LIVE_VERSION" = "$TARGET_VERSION" ]; then
+                echo "✅ Success! Production has successfully rolled out $TARGET_VERSION ($TARGET_COMMIT)."
+                exit 0
+              fi
+            else
+              echo "  Endpoint did not return valid JSON or timed out."
+            fi
+
+            ATTEMPT=$((ATTEMPT + 1))
+            sleep 10
+          done
+
+          echo "❌ Deployment verification timed out after 5 minutes."
+          exit 1

ci/config/nginx/default.conf

@@ -44,6 +44,13 @@ server {
         try_files $uri $uri/ $uri.html /ru/index.html;
     }
 
+    # Version probe for release rollout verification
+    location = /version.json {
+        add_header Cache-Control "no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0" always;
+        add_header Content-Type "application/json" always;
+        try_files /version.json =404;
+    }
+
     # Error pages
     error_page 404 /en/index.html;
 }

ci/docker/docs/Dockerfile

@@ -14,8 +14,15 @@ RUN hugo --gc --minify
 # Stage 2: Runtime
 FROM nginx:alpine
 
+ARG COMMIT_SHA="unknown"
+ARG VERSION="latest"
+
 COPY --from=builder /src/services/docs/public /usr/share/nginx/html
 COPY ci/config/nginx/default.conf /etc/nginx/conf.d/default.conf
 
+# Generate version metadata for automated rollout verification
+RUN printf '{\n  "service": "docs",\n  "version": "%s",\n  "commit": "%s",\n  "builtAt": "%s"\n}\n' \
+    "${VERSION}" "${COMMIT_SHA}" "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" > /usr/share/nginx/html/version.json
+
 EXPOSE 80
 CMD ["nginx", "-g", "daemon off;"]

ci/docker/www/Dockerfile

@@ -37,8 +37,15 @@ RUN pnpm --filter @ermnvldmr/www build:all
 # Stage 2: Runtime
 FROM nginx:alpine
 
+ARG COMMIT_SHA="unknown"
+ARG VERSION="latest"
+
 COPY --from=builder /app/services/www/dist /usr/share/nginx/html
 COPY ci/config/nginx/default.conf /etc/nginx/conf.d/default.conf
 
+# Generate version metadata for automated rollout verification
+RUN printf '{\n  "service": "www",\n  "version": "%s",\n  "commit": "%s",\n  "builtAt": "%s"\n}\n' \
+    "${VERSION}" "${COMMIT_SHA}" "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" > /usr/share/nginx/html/version.json
+
 EXPOSE 80
 CMD ["nginx", "-g", "daemon off;"]