feat(ops): show read-replica health on the Operaciones screen
my.jorgecuadros.com serves customer balances from the Oracle VPS replica. A replica whose SQL thread has stopped does not error — it keeps answering, with data frozen at the moment it stopped — so nothing on the customer site looks wrong and the only signal is a customer complaining about a stale balance. This puts the failure somewhere a human sees it. Deliberately does not trust the two fields an operator reaches for first. Replica_IO_Running reports Yes while the SQL thread is stopped, because the network thread keeps downloading binlog it will never apply; verified by stopping SQL_THREAD and watching IO stay Yes. Seconds_Behind_Source reads NULL whenever EITHER thread is down, so the card renders "sin dato" rather than "0 s" — showing zero there would report an outage as perfect health. The problem string is resolved most-specific-first for the same reason. Shells out to the mysql client because the API has no MySQL driver and the image already ships one. --ssl is required (the replica sets require_secure_transport); --ssl-verify-server-cert=0 is deliberate and is NOT the trade-off the website makes: this hop never leaves Tailscale and the replica's firewall admits only this host, so WireGuard authenticates the peer, whereas the DreamHost leg crosses the public internet and pins the CA. The account behind it holds REPLICATION CLIENT and nothing else — it cannot read a single row. REPLICA_DB_* unset is a supported state and renders "no configurada", which is correct in dev and before cutover. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,153 @@
|
||||
import { Injectable, Logger } from "@nestjs/common";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
|
||||
export interface ReplicationStatus {
|
||||
/** false when the replica is not configured for this environment at all. */
|
||||
configured: boolean;
|
||||
/** true only when both threads run, no error is set, and lag is within bounds. */
|
||||
healthy: boolean;
|
||||
host: string | null;
|
||||
ioRunning: string | null;
|
||||
sqlRunning: string | null;
|
||||
/** null when MySQL reports NULL, which it does whenever a thread is down. */
|
||||
secondsBehind: number | null;
|
||||
lastIoError: string | null;
|
||||
lastSqlError: string | null;
|
||||
sourceHost: string | null;
|
||||
/** Human-readable reason when healthy is false. */
|
||||
problem: string | null;
|
||||
checkedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports whether the my.jorgecuadros.com read replica is still replicating.
|
||||
*
|
||||
* The replica is what the public site reads once the platformDataSource flag is
|
||||
* on, and a replica that has silently stopped applying serves stale balances
|
||||
* rather than erroring — the failure is invisible from the site itself, which is
|
||||
* why it needs a panel.
|
||||
*
|
||||
* Shells out to the mysql client for the same reason the rest of OpsService
|
||||
* does: there is no MySQL driver in this API's dependencies, and the image
|
||||
* already ships one.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ReplicationService {
|
||||
private readonly logger = new Logger(ReplicationService.name);
|
||||
|
||||
/** Lag above this many seconds is reported as unhealthy. */
|
||||
private readonly maxLagSeconds = Number(process.env.REPLICA_MAX_LAG ?? 60);
|
||||
|
||||
async status(): Promise<ReplicationStatus> {
|
||||
const host = process.env.REPLICA_DB_HOST;
|
||||
const user = process.env.REPLICA_DB_USER;
|
||||
const password = process.env.REPLICA_DB_PASS;
|
||||
const now = new Date().toISOString();
|
||||
|
||||
const empty: ReplicationStatus = {
|
||||
configured: false,
|
||||
healthy: false,
|
||||
host: host ?? null,
|
||||
ioRunning: null,
|
||||
sqlRunning: null,
|
||||
secondsBehind: null,
|
||||
lastIoError: null,
|
||||
lastSqlError: null,
|
||||
sourceHost: null,
|
||||
problem: null,
|
||||
checkedAt: now,
|
||||
};
|
||||
|
||||
if (!host || !user || !password) {
|
||||
return { ...empty, problem: "REPLICA_DB_* no configuradas" };
|
||||
}
|
||||
|
||||
let raw: string;
|
||||
try {
|
||||
// --ssl is required: the replica sets require_secure_transport=ON.
|
||||
//
|
||||
// --ssl-verify-server-cert=0 is deliberate and is NOT the same trade-off
|
||||
// the website makes. This hop never leaves Tailscale — the replica is
|
||||
// reached on its CGNAT tailnet address and its firewall admits only this
|
||||
// host — so WireGuard already authenticates the peer. The DreamHost leg
|
||||
// crosses the public internet and therefore pins the CA instead. The
|
||||
// client here is MariaDB's, which rejects our self-signed CA outright
|
||||
// unless it is handed the CA file, which would mean shipping a cert into
|
||||
// this image for a link that is already authenticated.
|
||||
const { stdout } = await exec(
|
||||
"mysql",
|
||||
[
|
||||
`--host=${host}`,
|
||||
`--user=${user}`,
|
||||
"--ssl",
|
||||
"--ssl-verify-server-cert=0",
|
||||
"--connect-timeout=5",
|
||||
"-e",
|
||||
"SHOW REPLICA STATUS\\G",
|
||||
],
|
||||
{
|
||||
env: { ...process.env, MYSQL_PWD: password },
|
||||
timeout: 15_000,
|
||||
},
|
||||
);
|
||||
raw = stdout;
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
this.logger.warn(`no se pudo consultar la réplica: ${msg}`);
|
||||
return { ...empty, configured: true, problem: `No se pudo conectar: ${msg}` };
|
||||
}
|
||||
|
||||
const field = (name: string): string | null => {
|
||||
const m = raw.match(new RegExp(`^\\s*${name}:\\s*(.*)$`, "m"));
|
||||
const v = m?.[1]?.trim();
|
||||
return v === undefined || v === "" ? null : v;
|
||||
};
|
||||
|
||||
// An empty result set means the server is not configured as a replica at
|
||||
// all — distinct from "configured but broken", and worth saying plainly.
|
||||
if (!raw.includes("Replica_IO_Running")) {
|
||||
return {
|
||||
...empty,
|
||||
configured: true,
|
||||
problem: "El servidor no está configurado como réplica",
|
||||
};
|
||||
}
|
||||
|
||||
const ioRunning = field("Replica_IO_Running");
|
||||
const sqlRunning = field("Replica_SQL_Running");
|
||||
const lagRaw = field("Seconds_Behind_Source");
|
||||
const secondsBehind =
|
||||
lagRaw === null || lagRaw === "NULL" ? null : Number(lagRaw);
|
||||
const lastIoError = field("Last_IO_Error");
|
||||
const lastSqlError = field("Last_SQL_Error");
|
||||
|
||||
// Order matters: report the most specific cause first. Checking lag before
|
||||
// the threads would blame "sin dato de retraso" for what is really a
|
||||
// stopped thread, because MySQL reports NULL lag whenever either is down.
|
||||
let problem: string | null = null;
|
||||
if (ioRunning !== "Yes") problem = "El hilo de E/S no está corriendo";
|
||||
else if (sqlRunning !== "Yes") problem = "El hilo SQL no está corriendo";
|
||||
else if (lastSqlError) problem = `Error SQL: ${lastSqlError}`;
|
||||
else if (lastIoError) problem = `Error de E/S: ${lastIoError}`;
|
||||
else if (secondsBehind === null) problem = "Sin dato de retraso";
|
||||
else if (secondsBehind > this.maxLagSeconds)
|
||||
problem = `Retraso de ${secondsBehind}s (máximo ${this.maxLagSeconds}s)`;
|
||||
|
||||
return {
|
||||
configured: true,
|
||||
healthy: problem === null,
|
||||
host,
|
||||
ioRunning,
|
||||
sqlRunning,
|
||||
secondsBehind,
|
||||
lastIoError,
|
||||
lastSqlError,
|
||||
sourceHost: field("Source_Host"),
|
||||
problem,
|
||||
checkedAt: now,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user