diff --git a/deploy/galactus/jorgecuadros-app.compose.yml b/deploy/galactus/jorgecuadros-app.compose.yml index 4b243f5..2018669 100644 --- a/deploy/galactus/jorgecuadros-app.compose.yml +++ b/deploy/galactus/jorgecuadros-app.compose.yml @@ -38,7 +38,12 @@ services: labels: io.jorgecuadros.role: "api" dns: + # MagicDNS first, then a public resolver. Listing ONLY 100.100.100.100 + # costs the container public name resolution — apk/npm/any outbound + # hostname stops resolving — because MagicDNS does not forward to an + # upstream unless the tailnet is configured with global nameservers. - ${TAILSCALE_DNS:-100.100.100.100} + - ${FALLBACK_DNS:-1.1.1.1} dns_search: - ${TAILNET_SUFFIX:-tail01aa2.ts.net} environment: @@ -83,7 +88,12 @@ services: # Next server-side rendering can call the API by API_ORIGIN, which is the # same MagicDNS name — so the web container needs the resolver too. dns: + # MagicDNS first, then a public resolver. Listing ONLY 100.100.100.100 + # costs the container public name resolution — apk/npm/any outbound + # hostname stops resolving — because MagicDNS does not forward to an + # upstream unless the tailnet is configured with global nameservers. - ${TAILSCALE_DNS:-100.100.100.100} + - ${FALLBACK_DNS:-1.1.1.1} dns_search: - ${TAILNET_SUFFIX:-tail01aa2.ts.net} environment: diff --git a/deploy/scripts/pre-migrate-backup.mjs b/deploy/scripts/pre-migrate-backup.mjs index 3218c56..e02a9a3 100644 --- a/deploy/scripts/pre-migrate-backup.mjs +++ b/deploy/scripts/pre-migrate-backup.mjs @@ -148,6 +148,34 @@ async function execInContainer(containerId, cmd, env = []) { } } +/** + * Run a SHORT command and return its combined output. + * + * Detached exec cannot report why anything failed, which once reduced a real + * failure to the single line "mysqldump exited 2" and cost a manual + * reproduction on the host to discover it was a missing auth plugin. Tty:true + * makes the start response a plain (non-multiplexed) stream that can just be + * read, at the cost of holding the connection open — fine for reading a small + * error file, which is why the dump itself still runs detached. + */ +async function execCapture(containerId, cmd) { + const created = await docker(`/containers/${containerId}/exec`, { + method: "POST", + body: JSON.stringify({ + AttachStdout: true, + AttachStderr: true, + Tty: true, + Cmd: ["sh", "-c", cmd], + }), + }); + const res = await fetch(`${DOCKER}/exec/${created.Id}/start`, { + method: "POST", + headers: { "X-API-Key": API_KEY, "Content-Type": "application/json" }, + body: JSON.stringify({ Detach: false, Tty: true }), + }); + return (await res.text()).trim(); +} + async function main() { const container = await findApiContainer(); if (!container) { @@ -173,17 +201,27 @@ async function main() { // Password via MYSQL_PWD in the exec env, never on the command line — argv is // world-readable through `ps` inside the container. const flags = `--host=${conn.host} --port=${conn.port} --user=${shq(conn.user)}`; + const errFile = "/tmp/pre-migrate-backup.err"; const dump = `set -o pipefail; mysqldump ${flags} --single-transaction --routines ` + - `--triggers --no-tablespaces ${shq(conn.database)} | gzip -c > ${shq(out)}`; + `--triggers --no-tablespaces ${shq(conn.database)} 2>${errFile} ` + + `| gzip -c > ${shq(out)}`; const code = await execInContainer(container.Id, dump, [ `MYSQL_PWD=${conn.password}`, ]); if (code !== 0) { - // Leave the truncated file behind for inspection but never let the deploy - // proceed believing it has a restore point. - throw new Error(`mysqldump exited ${code} — refusing to migrate`); + // An exit code alone is not actionable — surface what mysqldump actually + // said. Leave the truncated file behind for inspection, but never let the + // deploy proceed believing it has a restore point. + const stderr = await execCapture( + container.Id, + `tail -20 ${errFile} 2>/dev/null`, + ); + throw new Error( + `mysqldump exited ${code} — refusing to migrate` + + (stderr ? `\n--- mysqldump stderr ---\n${stderr}` : ""), + ); } // Detached exec gives no stdout, so prove the artefact separately: non-empty diff --git a/docker/api.Dockerfile b/docker/api.Dockerfile index 711bce6..c5c47fb 100644 --- a/docker/api.Dockerfile +++ b/docker/api.Dockerfile @@ -38,7 +38,15 @@ ENV NODE_ENV=production # runtime (linux-musl-openssl-3.0.x) and aborts with "Please manually install # OpenSSL" without it. Node bundles its own OpenSSL, so nothing else in this # image pulls the system package in. -RUN apk add --no-cache python3 mdbtools mysql-client openssl \ +# mariadb-connector-c is REQUIRED, not incidental. Alpine's `mysql-client` is +# MariaDB's client, and it ships with an EMPTY /usr/lib/mariadb/plugin — so it +# cannot perform caching_sha2_password, which is MySQL 8.4's default and +# effectively only auth method. Without this package every mysqldump/mysql call +# from the container dies with: +# ERROR 1045: Plugin caching_sha2_password could not be loaded +# That breaks the pre-migrate deploy backup AND the whole "Operaciones" admin +# panel (backup, restore, sync, re-import all shell out to these binaries). +RUN apk add --no-cache python3 mdbtools mysql-client mariadb-connector-c openssl \ && apk add --no-cache --virtual .pybuild python3-dev build-base \ && rm -rf /var/cache/apk/* diff --git a/docs/DEPLOY_AND_MIGRATIONS.md b/docs/DEPLOY_AND_MIGRATIONS.md index 4706b67..260b452 100644 --- a/docs/DEPLOY_AND_MIGRATIONS.md +++ b/docs/DEPLOY_AND_MIGRATIONS.md @@ -232,6 +232,27 @@ moment the app is served over TLS or reachable off-tailnet. Behind a TLS-terminating reverse proxy, set `trust proxy` on the Nest app instead of disabling the flag. +## The MySQL client inside the API image + +Alpine's `mysql-client` package is **MariaDB's** client, and it installs an +empty `/usr/lib/mariadb/plugin`. It therefore cannot speak +`caching_sha2_password`, which is MySQL 8.4's default and effectively only auth +method, and every `mysqldump`/`mysql` call from the container fails with: + +``` +ERROR 1045: Plugin caching_sha2_password could not be loaded: + ... /usr/lib/mariadb/plugin/caching_sha2_password.so: No such file or directory +``` + +`mariadb-connector-c` supplies that plugin and is installed in +`docker/api.Dockerfile` for exactly this reason — do not drop it as an unused +dependency. It affects far more than the deploy backup: the entire +**Operaciones** panel (backup, restore, sync, re-import) shells out to these +binaries, so without it none of those work in a container either. The feature +had only ever been exercised with the API running on a developer machine, where +the Oracle client is installed, which is why this went unnoticed until the +first containerised deploy. + ## Known caveats in the deploy path - The pre-migrate backup step sets `NODE_TLS_REJECT_UNAUTHORIZED=0` because