diff --git a/apps/api/src/ops/ops.controller.ts b/apps/api/src/ops/ops.controller.ts index 129190f..d40c122 100644 --- a/apps/api/src/ops/ops.controller.ts +++ b/apps/api/src/ops/ops.controller.ts @@ -104,6 +104,24 @@ export class OpsController { return this.replication.status(); } + /** + * Full row-by-row comparison of the customer-visible tables against the master. + * + * POST rather than GET despite reading nothing: it is a full scan of both + * servers and must not be something a browser prefetch, a retry, or a refresh + * can set off. Audited for the same reason — it is a deliberate, costly act, + * and "who ran this while the site was slow" is a question worth answering. + */ + @Post("replication/verify") + async verifyReplication(@Req() req: Request) { + const result = await this.replication.verify(); + void this.audit.log(this.actingId(req), "ops.replication.verify", { + identical: result.identical, + elapsedMs: result.elapsedMs, + }); + return result; + } + @Get("jobs") listJobs() { return this.ops.listJobs(); diff --git a/apps/api/src/ops/replication.service.ts b/apps/api/src/ops/replication.service.ts index 946f36b..81989d6 100644 --- a/apps/api/src/ops/replication.service.ts +++ b/apps/api/src/ops/replication.service.ts @@ -43,6 +43,58 @@ export interface ApplyProgress { percent: number | null; } +/** + * How far the replica's executed history is from the master's, in transactions. + * + * This is the check `SHOW REPLICA STATUS` cannot give you, and it is stronger + * than everything else on the card for one specific reason: every other field is + * self-reported by the replica. `Seconds_Behind_Source` reads 0 both when there + * is genuinely nothing to apply AND when the I/O thread is disconnected — with + * no incoming event there is nothing to measure staleness against, so a dead + * link reports as perfectly current. `GTID_SUBTRACT(master, replica)` asks the + * master what it has done and the replica what it has applied, so a silent + * disconnect shows up immediately as a growing number. + */ +export interface GtidDrift { + /** Transactions the master executed that the replica has not. 0 = identical. */ + missingTransactions: number; + /** The missing GTID set verbatim. Null when nothing is missing. */ + missingGtidSet: string | null; + /** + * Transactions in the replica's `gtid_executed` under its OWN server UUID — + * writes that happened here and exist nowhere on the master. + * + * Reported, never alarmed on. A non-zero count is the expected residue of the + * seed load: restoring a dump executes its statements locally, and they take + * GTIDs from this server's UUID. They never propagate (`log_replica_updates` + * is off and nothing sources from this node), so they are harmless — right up + * until someone tries to promote this box, where they become a real divergence. + */ + localTransactions: number; +} + +/** One table's row count and content fingerprint, on one side of the link. */ +export interface TableFingerprint { + table: string; + masterRows: number; + replicaRows: number; + /** Order-independent checksum over every column of every row. */ + masterChecksum: string; + replicaChecksum: string; + matches: boolean; +} + +export interface VerifyResult { + /** True only when every table matched on both count and checksum. */ + identical: boolean; + tables: TableFingerprint[]; + /** Set instead of `tables` when the comparison could not be run at all. */ + problem: string | null; + checkedAt: string; + /** Wall-clock cost, because this is a full scan and the caller should see it. */ + elapsedMs: number; +} + export interface ReplicationStatus { /** false when the replica is not configured for this environment at all. */ configured: boolean; @@ -58,11 +110,31 @@ export interface ReplicationStatus { sourceHost: string | null; /** Relay-log apply progress. Null when the status output has no positions. */ apply: ApplyProgress | null; + /** GTID comparison against the master. Null when the master was unreachable. */ + drift: GtidDrift | null; /** Human-readable reason when healthy is false. */ problem: string | null; checkedAt: string; } +/** + * The tables `my.jorgecuadros.com` reads through the `web_reader` grant. + * + * This list is the verification surface, not the replication surface — the + * replica carries the whole schema. These are the eight whose divergence would + * actually be visible to a customer, so they are the ones worth a full scan. + */ +export const REPLICATED_TABLES = [ + "transactions", + "customers", + "customer_legacy_refs", + "type_transactions", + "exchange_rates", + "properties", + "property_services", + "trust_accounts", +] as const; + /** * Reports whether the my.jorgecuadros.com read replica is still replicating. * @@ -99,6 +171,7 @@ export class ReplicationService { lastSqlError: null, sourceHost: null, apply: null, + drift: null, problem: null, checkedAt: now, }; @@ -109,33 +182,7 @@ export class ReplicationService { let raw: string; try { - // --ssl is required: the replica sets require_secure_transport=ON. - // - // --ssl-verify-server-cert=0 is deliberate and is NOT the same trade-off - // the website makes. This hop never leaves Tailscale — the replica is - // reached on its CGNAT tailnet address and its firewall admits only this - // host — so WireGuard already authenticates the peer. The DreamHost leg - // crosses the public internet and therefore pins the CA instead. The - // client here is MariaDB's, which rejects our self-signed CA outright - // unless it is handed the CA file, which would mean shipping a cert into - // this image for a link that is already authenticated. - const { stdout } = await exec( - "mysql", - [ - `--host=${host}`, - `--user=${user}`, - "--ssl", - "--ssl-verify-server-cert=0", - "--connect-timeout=5", - "-e", - "SHOW REPLICA STATUS\\G", - ], - { - env: { ...process.env, MYSQL_PWD: password }, - timeout: 15_000, - }, - ); - raw = stdout; + raw = await this.onReplica("SHOW REPLICA STATUS\\G"); } catch (e) { const msg = e instanceof Error ? e.message : String(e); this.logger.warn(`no se pudo consultar la réplica: ${msg}`); @@ -189,10 +236,341 @@ export class ReplicationService { // alarming on it would cry wolf. It is here to answer "is it moving?" // when the lag counter is stuck. apply: applyProgress(raw), + // Also reported rather than alarmed on, for the same reason: a busy master + // is always a few transactions ahead for the instant they are in flight. + // Null rather than zero when the master could not be reached — "unknown" + // and "identical" must not render the same. + drift: await this.gtidDrift(), problem, checkedAt: now, }; } + + /** + * Compare executed history between master and replica. + * + * Two round trips: ask the master what it has executed, then ask the replica + * to subtract its own history from that. The subtraction runs on the replica + * rather than in TypeScript because `GTID_SUBTRACT` already implements the + * interval algebra correctly, and reimplementing set subtraction over binlog + * ranges is exactly the kind of thing that looks right and is wrong at the + * boundaries. + * + * @returns null on any failure — a broken drift check must never be mistaken + * for a healthy zero. + */ + private async gtidDrift(): Promise { + try { + const masterGtid = (await this.onMaster("SELECT @@gtid_executed")).trim(); + + // GTID sets are UUIDs, digits, colons, commas, hyphens, whitespace and + // (since 8.4) alphanumeric tags. Nothing else is legal, so rejecting + // anything outside that alphabet is a whitelist, not a blacklist: with no + // quote and no backslash able to survive it, the value cannot escape the + // string literal it is interpolated into below. + if (masterGtid && !/^[0-9a-fA-F:,\s_-]+$/.test(masterGtid)) { + this.logger.warn("gtid_executed del maestro con formato inesperado"); + return null; + } + // An empty set means the master has GTID mode off, and there is nothing + // meaningful to compare. + if (!masterGtid) return null; + + const flat = masterGtid.replace(/\s+/g, ""); + // Every GTID set is flattened with REPLACE before it leaves the server. + // MySQL wraps `gtid_executed` across lines once it holds more than one + // source UUID, and this is read back as tab-separated columns — an + // embedded newline would split one row into two and silently truncate the + // set at the first UUID. + const out = await this.onReplica( + "SELECT REPLACE(GTID_SUBTRACT(" + + `'${flat}', @@gtid_executed), '\\n', ''), ` + + "@@server_uuid, REPLACE(@@gtid_executed, '\\n', '')", + ["-N"], + ); + + // Trailing newline only — never `.trim()`. When nothing is missing the + // first column is the empty string, so the line begins with a tab, and + // trimming it would shift every column one position left and report the + // replica's own UUID as the missing GTID set. + const [missingSet = "", serverUuid = "", executed = ""] = out + .replace(/\r?\n+$/, "") + .split("\t"); + + return { + missingTransactions: countGtids(missingSet), + missingGtidSet: missingSet || null, + localTransactions: countGtids(gtidsForUuid(executed, serverUuid)), + }; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + this.logger.warn(`no se pudo comparar GTIDs con el maestro: ${msg}`); + return null; + } + } + + /** + * Full-scan comparison of the customer-visible tables on both sides. + * + * Deliberately NOT part of `status()`: this reads every row of every table in + * `REPLICATED_TABLES` on both servers, so it belongs behind a button, not a + * 30-second poll. + * + * It answers the one question GTID drift cannot. GTIDs prove the replica + * applied every transaction the master produced; they say nothing about rows + * changed on the replica by some other route. A local write is invisible to + * every other field on the card and shows up here as a checksum mismatch. + */ + async verify(): Promise { + const started = Date.now(); + const base: VerifyResult = { + identical: false, + tables: [], + problem: null, + checkedAt: new Date().toISOString(), + elapsedMs: 0, + }; + + let sql: string; + try { + sql = await this.fingerprintSql(); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return { ...base, problem: `No se pudo leer el esquema: ${msg}`, elapsedMs: Date.now() - started }; + } + + let masterOut: string; + let replicaOut: string; + try { + // Sequential, not parallel. Running both at once would have the master + // scan under the replica's own read load only sometimes, which makes a + // slow run hard to attribute; and the boxes are small enough that two + // concurrent full scans is a real memory event on the 946MB replica. + masterOut = await this.onMaster(sql); + replicaOut = await this.onReplica(sql); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return { ...base, problem: `No se pudo comparar: ${msg}`, elapsedMs: Date.now() - started }; + } + + const master = parseFingerprints(masterOut); + const replica = parseFingerprints(replicaOut); + + const tables: TableFingerprint[] = REPLICATED_TABLES.map((table) => { + const m = master.get(table); + const r = replica.get(table); + return { + table, + masterRows: m?.rows ?? -1, + replicaRows: r?.rows ?? -1, + masterChecksum: m?.checksum ?? "?", + replicaChecksum: r?.checksum ?? "?", + // Both sides must have answered. A missing row on either side is a + // mismatch, never a pass — `undefined === undefined` would otherwise + // report two failed reads as agreement. + matches: + m !== undefined && r !== undefined && m.rows === r.rows && m.checksum === r.checksum, + }; + }); + + return { + identical: tables.every((t) => t.matches), + tables, + problem: null, + checkedAt: base.checkedAt, + elapsedMs: Date.now() - started, + }; + } + + /** + * Build the count+checksum query from the live column list. + * + * The columns come from `information_schema` on the master rather than being + * hardcoded, so the check keeps covering the whole row after a migration adds + * one. Reading the schema from the master is safe by construction: if the two + * schemas had diverged, replication would already be broken. + */ + private async fingerprintSql(): Promise { + const list = REPLICATED_TABLES.map((t) => `'${t}'`).join(","); + const raw = await this.onMaster( + "SELECT CONCAT(TABLE_NAME, '\\t', COLUMN_NAME) FROM information_schema.COLUMNS " + + `WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (${list}) ` + + "ORDER BY TABLE_NAME, ORDINAL_POSITION", + ["-N"], + ); + + const cols = new Map(); + for (const line of raw.split("\n")) { + const [table, column] = line.trim().split("\t"); + if (!table || !column) continue; + cols.set(table, [...(cols.get(table) ?? []), column]); + } + + const selects = REPLICATED_TABLES.map((table) => { + const columns = cols.get(table); + if (!columns?.length) throw new Error(`tabla ${table} sin columnas`); + // CONVERT(... USING binary), never CAST(... AS CHAR). + // + // CAST to CHAR transcodes into the *connection* character set, which is + // not the same on the two servers: the mysql client inside the master's + // container negotiates latin1, while the replica's negotiates utf8mb4. + // Every accented character in a Mexican name, street or note therefore + // hashes to different bytes on each side, and the comparison reports a + // permanent mismatch on exactly the tables that hold free text — a + // verification tool that always cries wolf, which is worse than none. + // Comparing the stored bytes sidesteps the session entirely. (Verified + // 2026-08-06: with CAST, `customers.name` gave 3344437324815 vs + // 3339150372121; with CONVERT both give 3339150372121.) + // + // 0x1f (unit separator) joins the columns and 0x1e (record separator) + // stands in for NULL. Both matter: CONCAT_WS *skips* NULLs rather than + // emitting an empty field, so without a placeholder the rows + // ('a', NULL, 'b') and ('a', 'b', NULL) produce the same string and a + // column-shifting bug would checksum as identical. + const expr = columns + .map((c) => `IFNULL(CONVERT(\`${c}\` USING binary), 0x1e)`) + .join(", 0x1f, "); + // SUM, not a running hash: addition is commutative, so the result does not + // depend on the order rows come back in. The two servers have no reason to + // scan in the same order and are not asked to. + return ( + `SELECT '${table}' AS t, COUNT(*) AS n, ` + + `IFNULL(SUM(CRC32(CONCAT_WS(0x1f, ${expr}))), 0) AS c FROM \`${table}\`` + ); + }); + + return selects.join(" UNION ALL "); + } + + /* ------------------------------------------------------------ plumbing */ + + /** + * Run a statement on the replica. + * + * --ssl is required: the replica sets require_secure_transport=ON. + * + * --ssl-verify-server-cert=0 is deliberate and is NOT the same trade-off the + * website makes. This hop never leaves Tailscale — the replica is reached on + * its CGNAT tailnet address and the tailnet ACL admits only this host — so + * WireGuard already authenticates the peer. The DreamHost leg crosses the + * public internet and therefore pins the CA instead. The client here is + * MariaDB's, which rejects our self-signed CA outright unless it is handed the + * CA file, which would mean shipping a cert into this image for a link that is + * already authenticated. + */ + private async onReplica(sql: string, extra: string[] = []): Promise { + const host = process.env.REPLICA_DB_HOST!; + const user = process.env.REPLICA_DB_USER!; + const password = process.env.REPLICA_DB_PASS!; + const { stdout } = await exec( + "mysql", + [ + `--host=${host}`, + `--user=${user}`, + "--ssl", + "--ssl-verify-server-cert=0", + "--connect-timeout=5", + ...extra, + "-e", + sql, + ], + { env: { ...process.env, MYSQL_PWD: password }, timeout: VERIFY_TIMEOUT_MS }, + ); + return stdout; + } + + /** + * Run a statement on the master, using the application's own DATABASE_URL. + * + * The app credential is enough here on purpose — everything this class sends + * to the master is a SELECT against `information_schema` or a system variable. + * Reaching for OPS_DB_ADMIN_* the way OpsService does would hand a monitoring + * read path a credential that can also restore a dump. + */ + private async onMaster(sql: string, extra: string[] = []): Promise { + const raw = process.env.DATABASE_URL; + if (!raw) throw new Error("DATABASE_URL no está configurada"); + const u = new URL(raw); + const { stdout } = await exec( + "mysql", + [ + `--host=${u.hostname}`, + `--port=${u.port || "3306"}`, + `--user=${decodeURIComponent(u.username)}`, + "--connect-timeout=5", + ...extra, + "-N", + "-e", + sql, + u.pathname.replace(/^\//, ""), + ], + { + env: { ...process.env, MYSQL_PWD: decodeURIComponent(u.password) }, + timeout: VERIFY_TIMEOUT_MS, + }, + ); + return stdout; + } +} + +/** Full scans on a 1-vCPU replica are not fast; 15s would cut them off. */ +const VERIFY_TIMEOUT_MS = 120_000; + +/** Parse the `t\tn\tc` rows the fingerprint query emits under `mysql -N`. */ +function parseFingerprints(raw: string): Map { + const out = new Map(); + for (const line of raw.split("\n")) { + const [table, n, c] = line.trim().split("\t"); + if (!table || n === undefined || c === undefined) continue; + const rows = Number(n); + if (!Number.isFinite(rows)) continue; + // The checksum stays a string. Sums of CRC32 over 40k rows exceed 2^53, so + // parsing it as a number would round and make distinct tables compare equal. + out.set(table, { rows, checksum: c }); + } + return out; +} + +/** + * Count the transactions in a GTID set. + * + * Exported for testing. The format is `uuid[:tag]:interval[:interval]...`, + * comma-separated, where an interval is `N` or `N-M` inclusive at both ends — + * so `1-5` is five transactions, not four. + * + * MySQL 8.4 added an optional alphanumeric tag between the UUID and the first + * interval. It is skipped rather than parsed: any segment that is not a number + * or a number range is not an interval, whatever else it may be. + */ +export function countGtids(set: string): number { + if (!set.trim()) return 0; + let total = 0; + for (const group of set.split(",")) { + for (const part of group.trim().split(":").slice(1)) { + const m = /^(\d+)(?:-(\d+))?$/.exec(part.trim()); + if (!m) continue; + const from = Number(m[1]); + const to = m[2] === undefined ? from : Number(m[2]); + if (Number.isFinite(from) && Number.isFinite(to) && to >= from) total += to - from + 1; + } + } + return total; +} + +/** + * Narrow a GTID set to the intervals belonging to one server UUID. + * + * Exported for testing. Used to isolate the replica's own writes from the + * history it replicated, which are interleaved in the same `gtid_executed`. + */ +export function gtidsForUuid(set: string, uuid: string): string { + if (!uuid.trim()) return ""; + const wanted = uuid.trim().toLowerCase(); + return set + .split(",") + .map((g) => g.trim()) + .filter((g) => g.toLowerCase().startsWith(`${wanted}:`)) + .join(","); } /** diff --git a/apps/api/src/ops/replication.spec.ts b/apps/api/src/ops/replication.spec.ts index e026df2..a030a34 100644 --- a/apps/api/src/ops/replication.spec.ts +++ b/apps/api/src/ops/replication.spec.ts @@ -1,4 +1,9 @@ -import { applyProgress, replicaField } from "./replication.service"; +import { + applyProgress, + countGtids, + gtidsForUuid, + replicaField, +} from "./replication.service"; /** * Verbatim shape of `SHOW REPLICA STATUS\G` from the live replica, trimmed to @@ -178,3 +183,72 @@ describe("applyProgress", () => { expect(applyProgress(positions("binlog.000042", "NULL", "binlog.000042", 400))).toBeNull(); }); }); + +/** + * Real GTID sets from the live pair, captured 2026-08-06. The replica's own + * server UUID (3b103283…) carries the transactions the seed dump load executed + * locally; the master's UUID (defc34e2…) carries the replicated history. + */ +const REPLICA_EXECUTED = + "3b103283-8f15-11f1-a52b-020017027b33:1-513," + + "defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-525"; +const REPLICA_UUID = "3b103283-8f15-11f1-a52b-020017027b33"; + +describe("countGtids", () => { + it("counts an inclusive range at both ends", () => { + // 1-5 is five transactions. Off-by-one here understates the gap, which is + // the direction that hides a problem. + expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-5")).toBe(5); + }); + + it("counts a bare single transaction", () => { + expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:7")).toBe(1); + }); + + it("sums several intervals under one UUID", () => { + expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-5:8:10-12")).toBe(9); + }); + + it("sums across UUIDs, including the wrapped form MySQL prints", () => { + expect(countGtids(REPLICA_EXECUTED)).toBe(513 + 525); + // `gtid_executed` comes back wrapped once it holds more than one UUID. + expect(countGtids(REPLICA_EXECUTED.replace(",", ",\n"))).toBe(513 + 525); + }); + + /** An empty subtraction result is the caught-up case and must be zero. */ + it("returns 0 for an empty or blank set", () => { + expect(countGtids("")).toBe(0); + expect(countGtids(" \n ")).toBe(0); + }); + + /** + * MySQL 8.4 allows an alphanumeric tag between the UUID and the intervals. + * It is not an interval and must not be counted as one. + */ + it("skips a tag without counting it", () => { + expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:mytag:1-3")).toBe(3); + }); +}); + +describe("gtidsForUuid", () => { + it("isolates the replica's own transactions from the replicated history", () => { + expect(countGtids(gtidsForUuid(REPLICA_EXECUTED, REPLICA_UUID))).toBe(513); + }); + + it("returns nothing for a UUID that is not in the set", () => { + expect(gtidsForUuid(REPLICA_EXECUTED, "00000000-0000-0000-0000-000000000000")).toBe(""); + }); + + /** + * The colon matters. Without it a UUID prefix would match a longer UUID that + * merely starts the same way, and the replica's local writes would be + * over-reported. + */ + it("does not match on a bare prefix", () => { + expect(gtidsForUuid(REPLICA_EXECUTED, "3b103283")).toBe(""); + }); + + it("returns nothing when the UUID is blank", () => { + expect(gtidsForUuid(REPLICA_EXECUTED, "")).toBe(""); + }); +}); diff --git a/apps/web/src/app/operaciones/page.tsx b/apps/web/src/app/operaciones/page.tsx index 5a80a66..3d4a90a 100644 --- a/apps/web/src/app/operaciones/page.tsx +++ b/apps/web/src/app/operaciones/page.tsx @@ -20,6 +20,7 @@ import { listOpsJobs, startOpsJob, uploadIngest, + verifyReplication, } from "@/lib/api"; import type { UploadProgress } from "@/lib/api"; import type { @@ -28,7 +29,9 @@ import type { IngestFile, OpsJob, OpsJobKind, + GtidDrift, ReplicationStatus, + VerifyResult, } from "@/lib/types"; const INGEST_MAX_BYTES = 2 * 1024 * 1024 * 1024; @@ -631,6 +634,9 @@ function OpTile({ function ReplicationCard() { const [status, setStatus] = useState(null); const [failed, setFailed] = useState(false); + const [verify, setVerify] = useState(null); + const [verifying, setVerifying] = useState(false); + const [verifyError, setVerifyError] = useState(null); const load = useCallback(() => { getReplicationStatus() @@ -641,6 +647,17 @@ function ReplicationCard() { .catch(() => setFailed(true)); }, []); + const runVerify = useCallback(() => { + setVerifying(true); + setVerifyError(null); + verifyReplication() + .then(setVerify) + .catch((e: unknown) => + setVerifyError(e instanceof Error ? e.message : "No se pudo comparar."), + ) + .finally(() => setVerifying(false)); + }, []); + useEffect(() => { load(); const t = setInterval(load, 30_000); @@ -704,14 +721,144 @@ function ReplicationCard() { value={status.secondsBehind === null ? "sin dato" : `${status.secondsBehind} s`} /> + + + {/* Only worth showing when it is not zero, and even then as a note rather + than a warning: these are the seed load's own transactions, and they + are inert until someone tries to promote this box. */} + {status.drift !== null && status.drift.localTransactions > 0 && ( +

+ La réplica tiene {status.drift.localTransactions.toLocaleString("es-MX")} transacciones + propias (de la carga inicial). No se propagan y no afectan la lectura; sólo importarían + si este servidor pasara a ser maestro. +

+ )} + + ); } +/** + * Row-by-row comparison against the master, on demand. + * + * Separate from the polled fields because it costs a full scan of both servers. + * It is the only check here that can catch a row changed on the replica by + * something other than replication — the GTID and lag figures would both still + * read perfectly healthy in that case. + */ +function VerifyPanel({ + result, + running, + error, + onRun, +}: { + result: VerifyResult | null; + running: boolean; + error: string | null; + onRun: () => void; +}) { + return ( +
+
+ + Compara fila por fila las 8 tablas que lee el sitio de clientes. Recorre ambos + servidores por completo, así que tarda. + + +
+ + {error && ( +
+ {error} +
+ )} + + {result?.problem && ( +
+ {result.problem} +
+ )} + + {result && !result.problem && ( + <> +

+ + {result.identical ? "Idénticas" : "Hay diferencias"} + {" "} + + {formatDateTime(result.checkedAt)} · {(result.elapsedMs / 1000).toFixed(1)} s + +

+
+ + + + + + + + + + + {result.tables.map((t) => ( + + + {/* -1 is the sentinel for "that server did not answer for + this table", which is not the same as zero rows. */} + + + + + ))} + +
TablaMaestroRéplicaEstado
{t.table} + {t.masterRows < 0 ? "—" : t.masterRows.toLocaleString("es-MX")} + + {t.replicaRows < 0 ? "—" : t.replicaRows.toLocaleString("es-MX")} + + + {t.matches + ? "igual" + : t.masterRows !== t.replicaRows + ? "difieren en filas" + : "difieren en contenido"} + +
+
+ + )} +
+ ); +} + +/** + * Transactions the master has executed that the replica has not. + * + * Rendered as its own line rather than folded into the lag figure because the + * two disagree in exactly the case that matters: a disconnected I/O thread + * reports 0 seconds of lag (no event has arrived to be late) while this number + * climbs. + */ +function driftLabel(drift: GtidDrift | null): string { + // Null means the master could not be reached. Saying "al día" here would be a + // lie of the worst kind — it is the reading a broken check produces. + if (drift === null) return "sin dato"; + if (drift.missingTransactions === 0) return "al día"; + return `${drift.missingTransactions.toLocaleString("es-MX")} transacciones atrás`; +} + /** * Bytes the replica has fetched but not yet applied. * diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index bf4431f..5450f06 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -60,6 +60,7 @@ import type { OpsJob, OpsJobKind, ReplicationStatus, + VerifyResult, IngestFile, BackupFile, PropertyDetail, @@ -964,6 +965,18 @@ export function getReplicationStatus(): Promise { return apiFetch("/ops/replication"); } +/** + * Compare every customer-visible table against the master, row by row. + * + * Slow by nature — it is a full scan of both servers — so it is a button, not + * part of the poll. Answers the question replication status cannot: GTIDs prove + * the replica applied everything the master sent, not that nothing else changed + * the rows here. + */ +export function verifyReplication(): Promise { + return apiFetch("/ops/replication/verify", { method: "POST" }); +} + export function listOpsJobs(): Promise { return apiFetch("/ops/jobs"); } diff --git a/apps/web/src/lib/types.ts b/apps/web/src/lib/types.ts index 1389890..ed46811 100644 --- a/apps/web/src/lib/types.ts +++ b/apps/web/src/lib/types.ts @@ -101,10 +101,52 @@ export interface ReplicationStatus { lastSqlError: string | null; sourceHost: string | null; apply: ApplyProgress | null; + drift: GtidDrift | null; problem: string | null; checkedAt: string; } +/** + * Executed-history gap between master and replica, in transactions. + * + * The only field on the card that is not self-reported by the replica, and the + * only one that catches a silently disconnected I/O thread: with no incoming + * events, `secondsBehind` reads 0 because there is nothing to measure staleness + * against, so a dead link looks perfectly current. This number grows instead. + * + * Null when the master could not be reached — "unknown" must not render as + * "identical". + */ +export interface GtidDrift { + missingTransactions: number; + missingGtidSet: string | null; + /** + * Transactions written on the replica under its own server UUID, which exist + * nowhere on the master. Non-zero is expected — restoring the seed dump + * executed its statements locally — and harmless while nothing replicates + * from this node. + */ + localTransactions: number; +} + +/** One table compared on both sides of the link. */ +export interface TableFingerprint { + table: string; + masterRows: number; + replicaRows: number; + masterChecksum: string; + replicaChecksum: string; + matches: boolean; +} + +export interface VerifyResult { + identical: boolean; + tables: TableFingerprint[]; + problem: string | null; + checkedAt: string; + elapsedMs: number; +} + /** * Relay-log apply progress, in source binlog bytes. *