From fa38ff581e5e006ea38282c3c364b83f1c2336cb Mon Sep 17 00:00:00 2001 From: Ricardo Mancinas Date: Wed, 19 Aug 2026 20:32:52 -0700 Subject: [PATCH] fix(deploy): reclaim superseded images so the host stops filling up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every build pushes a new api + web image and every deploy pulls both onto galactus, but nothing ever removed the pair they replaced. That reached 63 images / 83.85GB, of which 79.26GB was unused, and filled the 98GB root filesystem to 100%. The symptom was not a disk alert. It was "re-import is broken": the Operaciones REIMPORT job leads with a mysqldump safety backup, that write had nowhere to go, and PIPEFAIL took the job down before it touched the database. Nothing in the ops_jobs log pointed at the disk. Prune runs last, after the verify step, because Docker refuses to prune an image that a container references — the running stack is what protects the release just shipped. `until` adds a grace window on top so a rollback dispatch stays a stack swap instead of a re-pull, but note it filters on image creation time rather than pull time, so it does NOT cover rolling back to an old tag; the running-container rule is what does. continue-on-error: housekeeping that fails leaves a fat host, not a broken release. Co-Authored-By: Claude Opus 5 --- .gitea/workflows/deploy-galactus.yml | 25 +++++++ deploy/scripts/prune-images.mjs | 106 +++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 deploy/scripts/prune-images.mjs diff --git a/.gitea/workflows/deploy-galactus.yml b/.gitea/workflows/deploy-galactus.yml index c9edd34..4212300 100644 --- a/.gitea/workflows/deploy-galactus.yml +++ b/.gitea/workflows/deploy-galactus.yml @@ -25,6 +25,10 @@ # through this workflow at all. # 4. app (api + web) the new images. # 5. verify ask the running API what it actually is. +# 6. prune images reclaim the superseded api/web images. LAST, and +# after verify: Docker will not prune an image a +# container references, so the running stack is what +# protects the release we just shipped. # # Rollback = re-dispatch with an older `tag`. That rolls back CODE only; the # schema stays forward. This is exactly why every schema change must be @@ -386,3 +390,24 @@ jobs: echo "dispatched '$WANT'; tiers report '$API_VER' (not directly comparable)" ;; esac + + # --- housekeeping ------------------------------------------------------ + # Runs LAST, and only after the verify step proved the new containers are + # up. See deploy/scripts/prune-images.mjs: Docker refuses to prune an + # image a container references, so "the stack is running" is what makes + # the current images safe. Pruning earlier would have nothing holding + # them. + # + # continue-on-error: reclaiming disk is not what the deploy is for. A + # prune that fails leaves a fat host, not a broken release. + - name: Prune unused images + continue-on-error: true + env: + PORTAINER_URL: ${{ secrets.PORTAINER_URL_GALACTUS }} + PORTAINER_API_KEY: ${{ secrets.PORTAINER_API_KEY_GALACTUS }} + PORTAINER_ENDPOINT_ID: ${{ secrets.PORTAINER_ENDPOINT_ID_GALACTUS }} + # Grace window. Keeps the previous few releases on disk so a rollback + # dispatch is a stack swap instead of a re-pull. + KEEP_HOURS: "168" + NODE_TLS_REJECT_UNAUTHORIZED: "0" + run: node deploy/scripts/prune-images.mjs diff --git a/deploy/scripts/prune-images.mjs b/deploy/scripts/prune-images.mjs new file mode 100644 index 0000000..c45dc37 --- /dev/null +++ b/deploy/scripts/prune-images.mjs @@ -0,0 +1,106 @@ +#!/usr/bin/env node +/** + * Delete unused images from the target host after a successful deploy. + * + * This exists because nothing else reclaims them. Every build.yml run pushes a + * new api + web image, every deploy pulls both onto the host, and the previous + * pair is left behind untagged-but-present forever. On galactus that reached + * 63 images / 83.85GB (79.26GB of it unused) and filled the 98GB root + * filesystem to 100% on 2026-08-20 — which surfaced as "re-import is broken", + * because the Operaciones REIMPORT job leads with a mysqldump that could no + * longer write its safety backup. + * + * Two things keep this from eating a live deployment: + * + * - Docker never prunes an image that a container references, running or + * stopped. The five images the prod stacks use are therefore untouchable + * for as long as their containers exist. + * - `until` gives a grace window on top of that, so a rollback target stays + * on disk instead of forcing a re-pull from the registry. + * + * TRAP: `until` filters on the image's CREATION time, not when the host pulled + * it. Rolling back to an old tag pulls an image that is already older than the + * window, so the grace period does NOT protect it — the running-container rule + * is what does. That is why this step must run AFTER the app stack is deployed + * and verified, never before. + * + * Required env: + * PORTAINER_URL, PORTAINER_API_KEY, PORTAINER_ENDPOINT_ID + * Optional: + * KEEP_HOURS grace window in hours (default 168 = 7 days) + * + * TLS: Portainer here is self-signed; the caller sets + * NODE_TLS_REJECT_UNAUTHORIZED=0 for this step. + */ + +function required(name) { + const v = process.env[name]; + if (!v) { + console.error(`missing required env: ${name}`); + process.exit(1); + } + return v; +} + +const PORTAINER_URL = required("PORTAINER_URL").replace(/\/+$/, ""); +const API_KEY = required("PORTAINER_API_KEY"); +const ENDPOINT_ID = required("PORTAINER_ENDPOINT_ID"); +const KEEP_HOURS = process.env.KEEP_HOURS || "168"; + +const DOCKER = `${PORTAINER_URL}/api/endpoints/${ENDPOINT_ID}/docker`; + +// `dangling: ["false"]` is what makes this `docker image prune -a` rather than +// the default, which only collects untagged layers. The tagged-but-superseded +// api/web images are the whole problem, and the default filter walks straight +// past them. +const FILTERS = JSON.stringify({ + dangling: ["false"], + until: [`${KEEP_HOURS}h`], +}); + +function human(bytes) { + if (!bytes) return "0B"; + const units = ["B", "KB", "MB", "GB", "TB"]; + let i = 0; + let n = bytes; + while (n >= 1024 && i < units.length - 1) { + n /= 1024; + i += 1; + } + return `${n.toFixed(i === 0 ? 0 : 2)}${units[i]}`; +} + +async function main() { + const url = `${DOCKER}/images/prune?filters=${encodeURIComponent(FILTERS)}`; + const res = await fetch(url, { + method: "POST", + headers: { "X-API-Key": API_KEY }, + }); + const body = await res.text(); + if (!res.ok) { + throw new Error(`prune -> HTTP ${res.status} ${body.slice(0, 300)}`); + } + + let report; + try { + report = JSON.parse(body); + } catch { + throw new Error(`prune returned non-JSON: ${body.slice(0, 300)}`); + } + + const deleted = report.ImagesDeleted ?? []; + const reclaimed = report.SpaceReclaimed ?? 0; + console.log( + `pruned images older than ${KEEP_HOURS}h and unused by any container`, + ); + console.log(` entries removed : ${deleted.length}`); + console.log(` space reclaimed : ${human(reclaimed)}`); +} + +main().catch((err) => { + // Non-fatal by contract: the step that calls this sets continue-on-error, so + // housekeeping never turns a good deploy red. Exit non-zero anyway so the + // failure is visible in the run rather than swallowed. + console.error(`::warning::image prune FAILED: ${err.message}`); + process.exit(1); +});