diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 4690100..b67c5cb 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -15,8 +15,15 @@ # container can report exactly what is deployed. # # Release flow: git tag v1.2.0 && git push origin v1.2.0 -> versioned images -# -> deploy-galactus.yml is dispatched automatically (see the -# `deploy` job at the bottom). +# -> deploy-on-tag.yml waits for this run to go green and then +# dispatches deploy-galactus.yml. +# +# This workflow BUILDS ONLY — it never deploys. The deploy chain used to be a +# job here, gated to tag refs, but Gitea draws every job of a workflow into the +# run graph before it evaluates the job's `if`: a routine master build showed a +# pending "Deploy to galactus" and looked like prod was about to be redeployed +# off an unreleased commit. Keeping the deploy in a `on: push: tags` workflow of +# its own makes that structurally impossible. name: Build and Push Images @@ -101,127 +108,3 @@ jobs: APP_VERSION=${{ steps.meta.outputs.version }} GIT_SHA=${{ github.sha }} BUILD_DATE=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} - - # Chain the PROD deploy onto a green tag build. - # - # `needs: build` waits for BOTH matrix legs, so api and web at this tag are - # both in the registry before anything is deployed — deploy-galactus.yml does - # not build, it only pulls, and a half-pushed pair is exactly the state that - # leaves prod running one new image and one old one. - # - # Why a dispatch and not a `workflow_run:` trigger, which Gitea does support - # as of 1.24: deploy-galactus.yml reads `github.event.inputs.*` in ten places - # (tag, scope, bootstrap, skip_migrate). Under workflow_run every one of them - # is the empty string, so the deploy would silently run with no tag and - # scope != 'full'. A dispatch keeps that workflow's contract intact and keeps - # it hand-runnable for rollbacks, which is the whole point of it. - # - # Kill switch: set the repo variable AUTO_DEPLOY_GALACTUS to `false` to cut - # the chain and go back to dispatching the deploy by hand. Anything else - # (including unset) deploys. - deploy: - name: Deploy to galactus - needs: build - if: startsWith(github.ref, 'refs/tags/v') - runs-on: docker - container: - image: node:20-alpine - steps: - - name: Preflight — RELEASE_TOKEN - env: - RELEASE_TOKEN: ${{ secrets.RELEASE_TOKEN }} - run: | - set -eu - if [ -z "${RELEASE_TOKEN:-}" ]; then - echo "::error::Secret RELEASE_TOKEN is not set, so this build cannot" - echo "::error::dispatch the deploy. The images ARE published — run" - V=${GITHUB_REF#refs/tags/} - echo "::error::'Deploy to galactus' by hand with tag=${V#v}." - exit 1 - fi - - - name: Dispatch deploy-galactus.yml - env: - RELEASE_TOKEN: ${{ secrets.RELEASE_TOKEN }} - AUTO_DEPLOY: ${{ vars.AUTO_DEPLOY_GALACTUS }} - TAG_REF: ${{ github.ref }} - run: | - node -e ' - const base = `${process.env.GITHUB_SERVER_URL}/api/v1/repos/${process.env.GITHUB_REPOSITORY}`; - const headers = { Authorization: `token ${process.env.RELEASE_TOKEN}` }; - // refs/tags/v1.2.3 — derived from github.ref rather than ref_name so - // it does not depend on how Gitea populates GITHUB_REF_NAME. - const tagRef = process.env.TAG_REF; - // The git tag carries the leading v; the image tag does not. - const version = tagRef.replace(/^refs\/tags\/v?/, ""); - const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); - - // Every deploy-galactus run id visible right now. A dispatch is - // only confirmed by an id that is NOT in here — a plain "is there a - // deploy run" check is satisfied by the previous release run, and - // would report success for a dispatch that never took. - const deployRunIds = async () => { - const r = await fetch(`${base}/actions/runs?limit=50`, { headers }); - if (!r.ok) throw new Error(`runs query failed: HTTP ${r.status}`); - const body = await r.json(); - return new Set( - (body.workflow_runs || []) - .filter((run) => String(run.path || "").includes("deploy-galactus.yml")) - .map((run) => run.id), - ); - }; - - (async () => { - if (process.env.AUTO_DEPLOY === "false") { - console.log("AUTO_DEPLOY_GALACTUS=false — not deploying."); - console.log(`Deploy by hand with tag=${version} when ready.`); - return; - } - - const before = await deployRunIds(); - - // Dispatch against the TAG, not master: the deploy applies the - // compose files under deploy/galactus/ from whatever ref it runs - // on, and those must be the ones this release was cut with. - const res = await fetch( - `${base}/actions/workflows/deploy-galactus.yml/dispatches`, - { - method: "POST", - headers: { ...headers, "Content-Type": "application/json" }, - body: JSON.stringify({ - ref: tagRef, - inputs: { - tag: version, - scope: "app", - bootstrap: "false", - skip_migrate: "false", - }, - }), - }, - ); - if (!res.ok) { - console.log(`::error::Dispatch returned HTTP ${res.status}: ${await res.text()}`); - console.log(`::error::Images for ${version} are published. Run`); - console.log(`::error::"Deploy to galactus" by hand with tag=${version}.`); - process.exit(1); - } - - // A 204 only means Gitea accepted the request. Confirm a NEW run - // exists, the same way release.yml confirms this build started — - // an accepted call that creates no run is the failure mode that - // cost v1.0.3 its images. - for (let i = 0; i < 3; i++) { - await sleep(5_000); - const now = await deployRunIds(); - const fresh = [...now].filter((id) => !before.has(id)); - if (fresh.length) { - console.log(`Deploy of ${version} to galactus is running (run ${fresh[0]}).`); - return; - } - } - - console.log(`::error::Dispatch was accepted but no deploy run appeared.`); - console.log(`::error::Run "Deploy to galactus" by hand with tag=${version}.`); - process.exit(1); - })(); - ' diff --git a/.gitea/workflows/deploy-on-tag.yml b/.gitea/workflows/deploy-on-tag.yml new file mode 100644 index 0000000..68b14c8 --- /dev/null +++ b/.gitea/workflows/deploy-on-tag.yml @@ -0,0 +1,199 @@ +# Chain the PROD deploy onto a green tag build. +# +# This is a SEPARATE workflow, not a job in build.yml, and the trigger is the +# whole point: `on: push: tags` cannot fire on a push to master. When this was a +# `deploy` job inside build.yml gated by `if: startsWith(github.ref, +# 'refs/tags/v')`, Gitea still drew "Deploy to galactus" into the job graph of +# every ordinary master build — the `if` is not evaluated until `needs` resolve, +# so the job sits there looking like an imminent production deploy on a commit +# nobody released. That is indistinguishable from a real misfire, and the only +# safe reaction is to cancel the run, which kills the images with it. +# +# What it does NOT do is build. build.yml already builds and pushes both images +# from one run; this waits for that run to go green and then dispatches +# deploy-galactus.yml, which only pulls. +# +# Why wait for the build run rather than just dispatching: deploy-galactus.yml +# pulls api and web at the same tag, and a half-pushed pair is exactly the state +# that leaves prod running one new image and one old one. The build run turning +# green is the signal that both are in the registry. +# +# Why a dispatch and not a `workflow_run:` trigger, which Gitea does support as +# of 1.24: deploy-galactus.yml reads `github.event.inputs.*` in ten places (tag, +# scope, bootstrap, skip_migrate). Under workflow_run every one of them is the +# empty string, so the deploy would silently run with no tag and scope != 'full'. +# A dispatch keeps that workflow's contract intact and keeps it hand-runnable for +# rollbacks, which is the whole point of it. +# +# Kill switch: set the repo variable AUTO_DEPLOY_GALACTUS to `false` to cut the +# chain and go back to dispatching the deploy by hand. Anything else (including +# unset) deploys. + +name: Deploy on tag + +on: + push: + tags: ["v*"] + +jobs: + deploy: + name: Deploy to galactus + runs-on: docker + container: + image: node:20-alpine + steps: + - name: Preflight — RELEASE_TOKEN + env: + RELEASE_TOKEN: ${{ secrets.RELEASE_TOKEN }} + run: | + set -eu + if [ -z "${RELEASE_TOKEN:-}" ]; then + echo "::error::Secret RELEASE_TOKEN is not set, so this cannot wait" + echo "::error::for the build or dispatch the deploy. Once build.yml" + V=${GITHUB_REF#refs/tags/} + echo "::error::is green, run 'Deploy to galactus' by hand with tag=${V#v}." + exit 1 + fi + + - name: Wait for the tag build, then dispatch deploy-galactus.yml + env: + RELEASE_TOKEN: ${{ secrets.RELEASE_TOKEN }} + AUTO_DEPLOY: ${{ vars.AUTO_DEPLOY_GALACTUS }} + TAG_REF: ${{ github.ref }} + BUILD_SHA: ${{ github.sha }} + run: | + node -e ' + const base = `${process.env.GITHUB_SERVER_URL}/api/v1/repos/${process.env.GITHUB_REPOSITORY}`; + const headers = { Authorization: `token ${process.env.RELEASE_TOKEN}` }; + // refs/tags/v1.2.3 — derived from github.ref rather than ref_name so + // it does not depend on how Gitea populates GITHUB_REF_NAME. + const tagRef = process.env.TAG_REF; + const tag = tagRef.replace(/^refs\/tags\//, ""); + // The git tag carries the leading v; the image tag does not. + const version = tag.replace(/^v/, ""); + const sha = process.env.BUILD_SHA; + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + + const runs = async () => { + const r = await fetch(`${base}/actions/runs?limit=50`, { headers }); + if (!r.ok) throw new Error(`runs query failed: HTTP ${r.status}`); + return (await r.json()).workflow_runs || []; + }; + + // The release commit and its tag are the SAME sha, and build.yml + // skips the master run by design — so a sha match alone can latch + // onto that skipped run and call the build green when no image was + // ever pushed. Require the tag ref when the API reports one. + const isTagRun = (r) => { + const ref = r.head_branch || r.ref || ""; + return !ref || ref === tag || ref === tagRef; + }; + + const buildRun = async () => + (await runs()).find( + (r) => + r.head_sha === sha && + String(r.path || "").includes("build.yml") && + isTagRun(r), + ); + + // Gitea reports a run as `status` and mirrors it into `conclusion`; + // read whichever is populated rather than betting on one field. + const outcome = (r) => String(r.conclusion || r.status || "").toLowerCase(); + const DONE = ["success", "failure", "cancelled", "canceled", "skipped"]; + + // Every deploy-galactus run id visible right now. A dispatch is only + // confirmed by an id that is NOT in here — a plain "is there a deploy + // run" check is satisfied by the PREVIOUS release run, and would + // report success for a dispatch that never took. + const deployRunIds = async () => + new Set( + (await runs()) + .filter((r) => String(r.path || "").includes("deploy-galactus.yml")) + .map((r) => r.id), + ); + + (async () => { + if (process.env.AUTO_DEPLOY === "false") { + console.log("AUTO_DEPLOY_GALACTUS=false — not deploying."); + console.log(`Deploy by hand with tag=${version} when ready.`); + return; + } + + // ~20 min. A build is about 90s; the rest is queue time behind + // other runs on a single runner. + let run = null; + for (let i = 0; i < 80; i++) { + run = await buildRun(); + if (run && DONE.includes(outcome(run))) break; + if (!run && i === 11) { + // Two minutes with no run at all. The post-receive hook drops + // runs silently when it errors (this cost v1.0.3 its images), + // so say so rather than timing out with no explanation. + console.log(`::warning::No build.yml run for ${tag} yet after 2 min.`); + console.log(`::warning::If the Gitea post-receive hook is broken, dispatch`); + console.log(`::warning::"Build and Push Images" by hand with ref=${tag}.`); + } + await sleep(15_000); + } + + if (!run) { + console.log(`::error::No build.yml run for ${tag} (${sha}) after 20 min.`); + console.log(`::error::Dispatch "Build and Push Images" with ref=${tag} (the`); + console.log(`::error::tag, not master), then deploy by hand with tag=${version}.`); + process.exit(1); + } + + const result = outcome(run); + if (result !== "success") { + console.log(`::error::build.yml for ${tag} ended as "${result}" — not deploying.`); + console.log(`::error::Fix the build, re-run it, then deploy by hand with tag=${version}.`); + process.exit(1); + } + + console.log(`build.yml for ${tag} is green (run ${run.id}). Deploying ${version}.`); + const before = await deployRunIds(); + + // Dispatch against the TAG, not master: the deploy applies the + // compose files under deploy/galactus/ from whatever ref it runs + // on, and those must be the ones this release was cut with. + const res = await fetch( + `${base}/actions/workflows/deploy-galactus.yml/dispatches`, + { + method: "POST", + headers: { ...headers, "Content-Type": "application/json" }, + body: JSON.stringify({ + ref: tagRef, + inputs: { + tag: version, + scope: "app", + bootstrap: "false", + skip_migrate: "false", + }, + }), + }, + ); + if (!res.ok) { + console.log(`::error::Dispatch returned HTTP ${res.status}: ${await res.text()}`); + console.log(`::error::Images for ${version} are published. Run`); + console.log(`::error::"Deploy to galactus" by hand with tag=${version}.`); + process.exit(1); + } + + // A 204 only means Gitea accepted the request. Confirm a NEW run + // exists — an accepted call that creates no run is the failure mode + // that cost v1.0.3 its images. + for (let i = 0; i < 3; i++) { + await sleep(5_000); + const fresh = [...(await deployRunIds())].filter((id) => !before.has(id)); + if (fresh.length) { + console.log(`Deploy of ${version} to galactus is running (run ${fresh[0]}).`); + return; + } + } + + console.log(`::error::Dispatch was accepted but no deploy run appeared.`); + console.log(`::error::Run "Deploy to galactus" by hand with tag=${version}.`); + process.exit(1); + })(); + ' diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 29e6478..5ccc17d 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -1,14 +1,16 @@ # Cut a release: stamp the version across every package.json, commit, tag, push. # # This does NOT build and does NOT deploy itself. Pushing the `vX.Y.Z` tag is -# what triggers build.yml, which publishes `X.Y.Z`, `X.Y`, `sha-` and -# `latest` image tags — and then, on a green tag build only, dispatches -# deploy-galactus.yml with `tag=X.Y.Z scope=app` (no leading v — the git tag -# carries the `v`, the image tag does not). +# what triggers both build.yml, which publishes the `X.Y.Z`, `X.Y`, +# `sha-` and `latest` image tags, and deploy-on-tag.yml, which waits for +# that build to go green and then dispatches deploy-galactus.yml with +# `tag=X.Y.Z scope=app` (no leading v — the git tag carries the `v`, the image +# tag does not). A tag is the only ref that starts either chain; pushing to +# master builds images and stops there. # # So cutting a release DOES reach prod. To cut a version without deploying it, -# set the repo variable AUTO_DEPLOY_GALACTUS=false first; build.yml's `deploy` -# job then prints the manual command instead of running it. +# set the repo variable AUTO_DEPLOY_GALACTUS=false first; deploy-on-tag.yml then +# prints the manual command instead of running it. # # Why a workflow instead of three local commands: the release commit is the one # thing that must be identical every time, and cutting it from a laptop is how @@ -270,8 +272,8 @@ jobs: echo "Released v${VERSION}." echo "" echo "build.yml is now building git.mancinas.io/rmancinas/jorgecuadros-{api,web}:${VERSION}." - echo "When both images are pushed, build.yml dispatches 'Deploy to galactus'" - echo "automatically with tag=${VERSION} scope=app bootstrap=false skip_migrate=false." + echo "deploy-on-tag.yml is watching that build; when it goes green it dispatches" + echo "'Deploy to galactus' with tag=${VERSION} scope=app bootstrap=false skip_migrate=false." echo "" echo "Watch that run. If it did not start (or AUTO_DEPLOY_GALACTUS=false)," echo "dispatch 'Deploy to galactus' by hand with the same inputs."