From 7e3b530174b1bd14a707ae0625e1c757a8b4d8fc Mon Sep 17 00:00:00 2001 From: Ricardo Mancinas Date: Thu, 30 Jul 2026 14:57:44 -0700 Subject: [PATCH] fix(deploy): pull images explicitly, and detect api/web drift by commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first successful galactus deploy came up all-green while the web tier was running a build from two commits earlier. The registry held web:latest from 3ff56e6; the host still had a web:latest cached from 4ee7ec7; the deploy reported success and served the old one. The API was only current because it had been pulled by hand during earlier debugging. Two independent failures, both fixed here. 1. Images are not pulled. The deploy action's `pull: true` does not reliably refresh an already-cached moving tag on a standalone endpoint. Added a Pull images step (deploy/scripts/pull-images.mjs) that pulls each image through Portainer's Docker API with registry credentials and fails the deploy if a pull fails — note the endpoint answers 200 even when the pull errored, so the stream body has to be inspected, not just the status. 2. The drift check could not see it. Both the verify step and the web footer compared APP_VERSION, but on a branch build BOTH tiers report "master", so equality proved nothing. They now compare gitSha, which is the only field that differs between two builds of the same branch. api and web come from one matrix run, so a difference can only mean an image was not replaced. This needed a /version on the web tier too — previously its build identity was only readable by scraping window.__APP_BUILD__ out of the HTML. pull-images.mjs builds the X-Registry-Auth header as URL-safe base64 WITH padding: Node's "base64url" omits the padding and Portainer's Go decoder rejects it with "Illegal base64 data at input byte N". Verified against galactus: pulls both images, and exits non-zero on a nonexistent tag. Co-Authored-By: Claude Opus 5 --- .gitea/workflows/deploy-galactus.yml | 71 +++++++++++++----- .gitea/workflows/deploy.yml | 62 ++++++++++++---- apps/web/src/app/version/route.ts | 18 +++++ apps/web/src/components/AppShell.tsx | 17 +++-- deploy/scripts/pull-images.mjs | 104 +++++++++++++++++++++++++++ 5 files changed, 236 insertions(+), 36 deletions(-) create mode 100644 apps/web/src/app/version/route.ts create mode 100644 deploy/scripts/pull-images.mjs diff --git a/.gitea/workflows/deploy-galactus.yml b/.gitea/workflows/deploy-galactus.yml index 78d180e..739d283 100644 --- a/.gitea/workflows/deploy-galactus.yml +++ b/.gitea/workflows/deploy-galactus.yml @@ -212,6 +212,23 @@ jobs: exit 1 fi + # --- make sure the host actually has the images ------------------------ + # The deploy action's `pull: true` does not reliably refresh an already + # cached moving tag. Pull explicitly, or a "successful" deploy can leave + # the host serving an older build of the same tag. + - name: Pull images + env: + PORTAINER_URL: ${{ secrets.PORTAINER_URL_GALACTUS }} + PORTAINER_API_KEY: ${{ secrets.PORTAINER_API_KEY_GALACTUS }} + PORTAINER_ENDPOINT_ID: ${{ secrets.PORTAINER_ENDPOINT_ID_GALACTUS }} + REGISTRY: ${{ env.REGISTRY }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + IMAGES: ${{ github.repository_owner }}/jorgecuadros-api,${{ github.repository_owner }}/jorgecuadros-web + TAG: ${{ github.event.inputs.tag }} + NODE_TLS_REJECT_UNAUTHORIZED: "0" + run: node deploy/scripts/pull-images.mjs + # --- always: the app (web + api) ------------------------------------- - name: Deploy app stack uses: cssnr/portainer-stack-deploy-action@v1 @@ -243,31 +260,53 @@ jobs: - name: Verify running version env: API_ORIGIN: ${{ secrets.APP_API_ORIGIN_GALACTUS }} + WEB_ORIGIN: ${{ secrets.APP_WEB_ORIGIN_GALACTUS }} WANT: ${{ github.event.inputs.tag }} - # The stack file naming a tag is not proof the container is running it — - # a skipped pull or a cached layer can leave the old code up. Ask it. + # A stack naming a tag is not proof the containers run it. Ask BOTH + # tiers what they are, and require them to be the same commit: api and + # web are built from one matrix run, so a difference can only mean one + # of them did not actually get replaced. run: | set -e apk add --no-cache curl >/dev/null - for i in $(seq 1 30); do - if curl -fsS "$API_ORIGIN/version" > /tmp/version.json; then break; fi - echo "waiting for API ($i/30)..." - sleep 5 - done - cat /tmp/version.json - GOT=$(node -e 'console.log(require("/tmp/version.json").version)') - # Only a semver dispatch is directly comparable: metadata-action's - # {{version}} turns tag v1.2.3 into image 1.2.3, while `latest` and - # `sha-*` report the branch or short sha instead. + fetch_version() { + for i in $(seq 1 30); do + if curl -fsS "$1/version" > "$2"; then return 0; fi + echo "waiting for $1 ($i/30)..." + sleep 5 + done + echo "::error::$1/version never answered" + return 1 + } + fetch_version "$API_ORIGIN" /tmp/api.json + fetch_version "$WEB_ORIGIN" /tmp/web.json + cat /tmp/api.json; echo; cat /tmp/web.json; echo + + API_SHA=$(node -e 'console.log(require("/tmp/api.json").gitSha)') + WEB_SHA=$(node -e 'console.log(require("/tmp/web.json").gitSha)') + API_VER=$(node -e 'console.log(require("/tmp/api.json").version)') + + # Compare the COMMIT, not the version string: on a branch build both + # tiers report "master", so version equality proves nothing. + if [ "$API_SHA" != "$WEB_SHA" ]; then + echo "::error::api and web are different builds — api $API_SHA, web $WEB_SHA" + echo "::error::one of the images was not replaced; check the Pull images step" + exit 1 + fi + echo "api and web agree: $API_SHA" + + # A semver dispatch is additionally comparable to the tag itself: + # metadata-action's {{version}} turns tag v1.2.3 into image 1.2.3, + # while `latest` and `sha-*` report the branch or short sha instead. case "$WANT" in [0-9]*.[0-9]*.[0-9]*) - if [ "$GOT" != "$WANT" ]; then - echo "::error::deployed $WANT but the API reports $GOT" + if [ "$API_VER" != "$WANT" ]; then + echo "::error::deployed $WANT but the API reports $API_VER" exit 1 fi - echo "verified: API is running $GOT" + echo "verified: running $API_VER" ;; *) - echo "dispatched '$WANT'; API reports '$GOT' (not directly comparable)" + echo "dispatched '$WANT'; tiers report '$API_VER' (not directly comparable)" ;; esac diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index c301e52..0443f90 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -221,6 +221,23 @@ jobs: exit 1 fi + # --- make sure the host actually has the images ------------------------ + # The deploy action's `pull: true` does not reliably refresh an already + # cached moving tag; without this a "successful" deploy can leave the host + # serving an older build of the same tag. + - name: Pull images + env: + PORTAINER_URL: ${{ secrets.PORTAINER_URL }} + PORTAINER_API_KEY: ${{ secrets.PORTAINER_API_KEY }} + PORTAINER_ENDPOINT_ID: ${{ secrets.PORTAINER_ENDPOINT_ID }} + REGISTRY: ${{ env.REGISTRY }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + IMAGES: ${{ github.repository_owner }}/jorgecuadros-api,${{ github.repository_owner }}/jorgecuadros-web + TAG: ${{ github.event.inputs.tag }} + NODE_TLS_REJECT_UNAUTHORIZED: "0" + run: node deploy/scripts/pull-images.mjs + # --- always: the app (web + api) ------------------------------------- - name: Deploy app stack uses: cssnr/portainer-stack-deploy-action@v1 @@ -253,29 +270,46 @@ jobs: - name: Verify running version env: API_ORIGIN: ${{ secrets.APP_API_ORIGIN }} + WEB_ORIGIN: ${{ secrets.APP_WEB_ORIGIN }} WANT: ${{ github.event.inputs.tag }} run: | set -e apk add --no-cache curl >/dev/null - for i in $(seq 1 30); do - if curl -fsS "$API_ORIGIN/version" > /tmp/version.json; then break; fi - echo "waiting for API ($i/30)..." - sleep 5 - done - cat /tmp/version.json - GOT=$(node -e 'console.log(require("/tmp/version.json").version)') - # Only a semver dispatch is directly comparable: metadata-action's - # {{version}} turns tag v1.2.3 into image 1.2.3, while `latest` and - # `sha-*` report the branch or short sha instead. + fetch_version() { + for i in $(seq 1 30); do + if curl -fsS "$1/version" > "$2"; then return 0; fi + echo "waiting for $1 ($i/30)..." + sleep 5 + done + echo "::error::$1/version never answered" + return 1 + } + fetch_version "$API_ORIGIN" /tmp/api.json + fetch_version "$WEB_ORIGIN" /tmp/web.json + cat /tmp/api.json; echo; cat /tmp/web.json; echo + + API_SHA=$(node -e 'console.log(require("/tmp/api.json").gitSha)') + WEB_SHA=$(node -e 'console.log(require("/tmp/web.json").gitSha)') + API_VER=$(node -e 'console.log(require("/tmp/api.json").version)') + + # Compare the COMMIT, not the version string: on a branch build both + # tiers report "master", so version equality proves nothing. + if [ "$API_SHA" != "$WEB_SHA" ]; then + echo "::error::api and web are different builds — api $API_SHA, web $WEB_SHA" + echo "::error::one of the images was not replaced; check the Pull images step" + exit 1 + fi + echo "api and web agree: $API_SHA" + case "$WANT" in [0-9]*.[0-9]*.[0-9]*) - if [ "$GOT" != "$WANT" ]; then - echo "::error::deployed $WANT but the API reports $GOT" + if [ "$API_VER" != "$WANT" ]; then + echo "::error::deployed $WANT but the API reports $API_VER" exit 1 fi - echo "verified: API is running $GOT" + echo "verified: running $API_VER" ;; *) - echo "dispatched '$WANT'; API reports '$GOT' (not directly comparable)" + echo "dispatched '$WANT'; tiers report '$API_VER' (not directly comparable)" ;; esac diff --git a/apps/web/src/app/version/route.ts b/apps/web/src/app/version/route.ts new file mode 100644 index 0000000..8624e68 --- /dev/null +++ b/apps/web/src/app/version/route.ts @@ -0,0 +1,18 @@ +import { NextResponse } from "next/server"; +import { readBuildInfoFromEnv } from "@/lib/build-info"; + +// Read per request, never prerendered — the whole point is to report what THIS +// running container is, and a baked answer would defeat that. +export const dynamic = "force-dynamic"; + +/** + * The web tier's counterpart to the API's GET /version. + * + * Without this, the only way to see what the web container is running was to + * scrape window.__APP_BUILD__ out of the HTML. The deploy workflow compares the + * two tiers' gitSha to catch a half-applied release, so it needs a stable, + * parseable answer from both sides. + */ +export function GET() { + return NextResponse.json({ service: "web", ...readBuildInfoFromEnv() }); +} diff --git a/apps/web/src/components/AppShell.tsx b/apps/web/src/components/AppShell.tsx index 57cea8d..0c816c9 100644 --- a/apps/web/src/components/AppShell.tsx +++ b/apps/web/src/components/AppShell.tsx @@ -3,7 +3,7 @@ import { useEffect, useRef, useState, type ReactNode } from "react"; import { usePathname, useRouter } from "next/navigation"; import Link from "next/link"; -import { getApiVersion, logout, me, updateUiScale } from "@/lib/api"; +import { getApiVersion, logout, me, updateUiScale, type ServiceVersion } from "@/lib/api"; import { shortSha, webBuildInfo } from "@/lib/build-info"; import { AuthContext, can } from "@/lib/abilities"; import { ROLE_LABEL } from "@/lib/labels"; @@ -190,13 +190,13 @@ function NavMenu({ */ function BuildFooter() { const web = webBuildInfo(); - const [api, setApi] = useState(null); + const [api, setApi] = useState(null); useEffect(() => { let alive = true; getApiVersion() .then((v) => { - if (alive) setApi(v.version); + if (alive) setApi(v); }) .catch(() => { // The shell already redirects to /login when the API is unreachable; @@ -207,7 +207,12 @@ function BuildFooter() { }; }, []); - const mismatch = api !== null && api !== web.version; + // Compare the COMMIT, not the version string. On a branch build both tiers + // report APP_VERSION "master", so comparing versions cannot see drift — which + // is exactly how a stale web image once sat next to a current API with this + // footer showing nothing wrong. The sha is the only field that actually + // differs between two builds of the same branch. + const mismatch = api !== null && api.gitSha !== web.gitSha; return (