feat(ops): verify the replica against the master, not just its own status
Every field the replication card showed was self-reported by the replica, and the two most reassuring ones lie in the same failure. Seconds_Behind_Source reads 0 when the I/O thread is disconnected — with no incoming event there is nothing to measure staleness against — and Replica_IO_Running only says the network thread is alive, not that it is receiving. Two checks that ask the master instead: - GTID drift, folded into the polled status. GTID_SUBTRACT(master, replica) counts transactions the master executed that the replica has not, so a silent disconnect shows up as a number that climbs instead of a lag that stays 0. It also isolates transactions carried under the replica's OWN server UUID — writes that exist nowhere on the master. There are currently 518 of them, residue of the seed dump load; inert while log_replica_updates is off, and a real divergence the day anyone promotes that box. - A full row-by-row comparison behind a button, over the eight tables my.jorgecuadros.com reads. GTIDs prove the replica applied everything the master sent; they say nothing about rows changed here by another route, which is the one failure the rest of the card cannot see. The comparison hashes CONVERT(col USING binary), not CAST(col AS CHAR). CAST transcodes into the connection character set, and the two servers do not agree on it: the client inside the master's container negotiates latin1, the replica's utf8mb4. Every accented character in a Mexican name, street or note then hashes differently and the tool reports a permanent mismatch on exactly the tables that hold free text. Caught by building it and running it — customers.name gave 3344437324815 against 3339150372121 under CAST, and 3339150372121 on both under CONVERT. All eight tables now match byte for byte. Verify is POST and audited despite reading nothing: it full-scans both servers, so a prefetch or a refresh must not be able to start one. Tests cover the GTID interval arithmetic, which is inclusive at both ends and easy to get wrong by one in the direction that hides a gap. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -104,6 +104,24 @@ export class OpsController {
|
||||
return this.replication.status();
|
||||
}
|
||||
|
||||
/**
|
||||
* Full row-by-row comparison of the customer-visible tables against the master.
|
||||
*
|
||||
* POST rather than GET despite reading nothing: it is a full scan of both
|
||||
* servers and must not be something a browser prefetch, a retry, or a refresh
|
||||
* can set off. Audited for the same reason — it is a deliberate, costly act,
|
||||
* and "who ran this while the site was slow" is a question worth answering.
|
||||
*/
|
||||
@Post("replication/verify")
|
||||
async verifyReplication(@Req() req: Request) {
|
||||
const result = await this.replication.verify();
|
||||
void this.audit.log(this.actingId(req), "ops.replication.verify", {
|
||||
identical: result.identical,
|
||||
elapsedMs: result.elapsedMs,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
@Get("jobs")
|
||||
listJobs() {
|
||||
return this.ops.listJobs();
|
||||
|
||||
@@ -43,6 +43,58 @@ export interface ApplyProgress {
|
||||
percent: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* How far the replica's executed history is from the master's, in transactions.
|
||||
*
|
||||
* This is the check `SHOW REPLICA STATUS` cannot give you, and it is stronger
|
||||
* than everything else on the card for one specific reason: every other field is
|
||||
* self-reported by the replica. `Seconds_Behind_Source` reads 0 both when there
|
||||
* is genuinely nothing to apply AND when the I/O thread is disconnected — with
|
||||
* no incoming event there is nothing to measure staleness against, so a dead
|
||||
* link reports as perfectly current. `GTID_SUBTRACT(master, replica)` asks the
|
||||
* master what it has done and the replica what it has applied, so a silent
|
||||
* disconnect shows up immediately as a growing number.
|
||||
*/
|
||||
export interface GtidDrift {
|
||||
/** Transactions the master executed that the replica has not. 0 = identical. */
|
||||
missingTransactions: number;
|
||||
/** The missing GTID set verbatim. Null when nothing is missing. */
|
||||
missingGtidSet: string | null;
|
||||
/**
|
||||
* Transactions in the replica's `gtid_executed` under its OWN server UUID —
|
||||
* writes that happened here and exist nowhere on the master.
|
||||
*
|
||||
* Reported, never alarmed on. A non-zero count is the expected residue of the
|
||||
* seed load: restoring a dump executes its statements locally, and they take
|
||||
* GTIDs from this server's UUID. They never propagate (`log_replica_updates`
|
||||
* is off and nothing sources from this node), so they are harmless — right up
|
||||
* until someone tries to promote this box, where they become a real divergence.
|
||||
*/
|
||||
localTransactions: number;
|
||||
}
|
||||
|
||||
/** One table's row count and content fingerprint, on one side of the link. */
|
||||
export interface TableFingerprint {
|
||||
table: string;
|
||||
masterRows: number;
|
||||
replicaRows: number;
|
||||
/** Order-independent checksum over every column of every row. */
|
||||
masterChecksum: string;
|
||||
replicaChecksum: string;
|
||||
matches: boolean;
|
||||
}
|
||||
|
||||
export interface VerifyResult {
|
||||
/** True only when every table matched on both count and checksum. */
|
||||
identical: boolean;
|
||||
tables: TableFingerprint[];
|
||||
/** Set instead of `tables` when the comparison could not be run at all. */
|
||||
problem: string | null;
|
||||
checkedAt: string;
|
||||
/** Wall-clock cost, because this is a full scan and the caller should see it. */
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
export interface ReplicationStatus {
|
||||
/** false when the replica is not configured for this environment at all. */
|
||||
configured: boolean;
|
||||
@@ -58,11 +110,31 @@ export interface ReplicationStatus {
|
||||
sourceHost: string | null;
|
||||
/** Relay-log apply progress. Null when the status output has no positions. */
|
||||
apply: ApplyProgress | null;
|
||||
/** GTID comparison against the master. Null when the master was unreachable. */
|
||||
drift: GtidDrift | null;
|
||||
/** Human-readable reason when healthy is false. */
|
||||
problem: string | null;
|
||||
checkedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The tables `my.jorgecuadros.com` reads through the `web_reader` grant.
|
||||
*
|
||||
* This list is the verification surface, not the replication surface — the
|
||||
* replica carries the whole schema. These are the eight whose divergence would
|
||||
* actually be visible to a customer, so they are the ones worth a full scan.
|
||||
*/
|
||||
export const REPLICATED_TABLES = [
|
||||
"transactions",
|
||||
"customers",
|
||||
"customer_legacy_refs",
|
||||
"type_transactions",
|
||||
"exchange_rates",
|
||||
"properties",
|
||||
"property_services",
|
||||
"trust_accounts",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Reports whether the my.jorgecuadros.com read replica is still replicating.
|
||||
*
|
||||
@@ -99,6 +171,7 @@ export class ReplicationService {
|
||||
lastSqlError: null,
|
||||
sourceHost: null,
|
||||
apply: null,
|
||||
drift: null,
|
||||
problem: null,
|
||||
checkedAt: now,
|
||||
};
|
||||
@@ -109,33 +182,7 @@ export class ReplicationService {
|
||||
|
||||
let raw: string;
|
||||
try {
|
||||
// --ssl is required: the replica sets require_secure_transport=ON.
|
||||
//
|
||||
// --ssl-verify-server-cert=0 is deliberate and is NOT the same trade-off
|
||||
// the website makes. This hop never leaves Tailscale — the replica is
|
||||
// reached on its CGNAT tailnet address and its firewall admits only this
|
||||
// host — so WireGuard already authenticates the peer. The DreamHost leg
|
||||
// crosses the public internet and therefore pins the CA instead. The
|
||||
// client here is MariaDB's, which rejects our self-signed CA outright
|
||||
// unless it is handed the CA file, which would mean shipping a cert into
|
||||
// this image for a link that is already authenticated.
|
||||
const { stdout } = await exec(
|
||||
"mysql",
|
||||
[
|
||||
`--host=${host}`,
|
||||
`--user=${user}`,
|
||||
"--ssl",
|
||||
"--ssl-verify-server-cert=0",
|
||||
"--connect-timeout=5",
|
||||
"-e",
|
||||
"SHOW REPLICA STATUS\\G",
|
||||
],
|
||||
{
|
||||
env: { ...process.env, MYSQL_PWD: password },
|
||||
timeout: 15_000,
|
||||
},
|
||||
);
|
||||
raw = stdout;
|
||||
raw = await this.onReplica("SHOW REPLICA STATUS\\G");
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
this.logger.warn(`no se pudo consultar la réplica: ${msg}`);
|
||||
@@ -189,10 +236,341 @@ export class ReplicationService {
|
||||
// alarming on it would cry wolf. It is here to answer "is it moving?"
|
||||
// when the lag counter is stuck.
|
||||
apply: applyProgress(raw),
|
||||
// Also reported rather than alarmed on, for the same reason: a busy master
|
||||
// is always a few transactions ahead for the instant they are in flight.
|
||||
// Null rather than zero when the master could not be reached — "unknown"
|
||||
// and "identical" must not render the same.
|
||||
drift: await this.gtidDrift(),
|
||||
problem,
|
||||
checkedAt: now,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare executed history between master and replica.
|
||||
*
|
||||
* Two round trips: ask the master what it has executed, then ask the replica
|
||||
* to subtract its own history from that. The subtraction runs on the replica
|
||||
* rather than in TypeScript because `GTID_SUBTRACT` already implements the
|
||||
* interval algebra correctly, and reimplementing set subtraction over binlog
|
||||
* ranges is exactly the kind of thing that looks right and is wrong at the
|
||||
* boundaries.
|
||||
*
|
||||
* @returns null on any failure — a broken drift check must never be mistaken
|
||||
* for a healthy zero.
|
||||
*/
|
||||
private async gtidDrift(): Promise<GtidDrift | null> {
|
||||
try {
|
||||
const masterGtid = (await this.onMaster("SELECT @@gtid_executed")).trim();
|
||||
|
||||
// GTID sets are UUIDs, digits, colons, commas, hyphens, whitespace and
|
||||
// (since 8.4) alphanumeric tags. Nothing else is legal, so rejecting
|
||||
// anything outside that alphabet is a whitelist, not a blacklist: with no
|
||||
// quote and no backslash able to survive it, the value cannot escape the
|
||||
// string literal it is interpolated into below.
|
||||
if (masterGtid && !/^[0-9a-fA-F:,\s_-]+$/.test(masterGtid)) {
|
||||
this.logger.warn("gtid_executed del maestro con formato inesperado");
|
||||
return null;
|
||||
}
|
||||
// An empty set means the master has GTID mode off, and there is nothing
|
||||
// meaningful to compare.
|
||||
if (!masterGtid) return null;
|
||||
|
||||
const flat = masterGtid.replace(/\s+/g, "");
|
||||
// Every GTID set is flattened with REPLACE before it leaves the server.
|
||||
// MySQL wraps `gtid_executed` across lines once it holds more than one
|
||||
// source UUID, and this is read back as tab-separated columns — an
|
||||
// embedded newline would split one row into two and silently truncate the
|
||||
// set at the first UUID.
|
||||
const out = await this.onReplica(
|
||||
"SELECT REPLACE(GTID_SUBTRACT(" +
|
||||
`'${flat}', @@gtid_executed), '\\n', ''), ` +
|
||||
"@@server_uuid, REPLACE(@@gtid_executed, '\\n', '')",
|
||||
["-N"],
|
||||
);
|
||||
|
||||
// Trailing newline only — never `.trim()`. When nothing is missing the
|
||||
// first column is the empty string, so the line begins with a tab, and
|
||||
// trimming it would shift every column one position left and report the
|
||||
// replica's own UUID as the missing GTID set.
|
||||
const [missingSet = "", serverUuid = "", executed = ""] = out
|
||||
.replace(/\r?\n+$/, "")
|
||||
.split("\t");
|
||||
|
||||
return {
|
||||
missingTransactions: countGtids(missingSet),
|
||||
missingGtidSet: missingSet || null,
|
||||
localTransactions: countGtids(gtidsForUuid(executed, serverUuid)),
|
||||
};
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
this.logger.warn(`no se pudo comparar GTIDs con el maestro: ${msg}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-scan comparison of the customer-visible tables on both sides.
|
||||
*
|
||||
* Deliberately NOT part of `status()`: this reads every row of every table in
|
||||
* `REPLICATED_TABLES` on both servers, so it belongs behind a button, not a
|
||||
* 30-second poll.
|
||||
*
|
||||
* It answers the one question GTID drift cannot. GTIDs prove the replica
|
||||
* applied every transaction the master produced; they say nothing about rows
|
||||
* changed on the replica by some other route. A local write is invisible to
|
||||
* every other field on the card and shows up here as a checksum mismatch.
|
||||
*/
|
||||
async verify(): Promise<VerifyResult> {
|
||||
const started = Date.now();
|
||||
const base: VerifyResult = {
|
||||
identical: false,
|
||||
tables: [],
|
||||
problem: null,
|
||||
checkedAt: new Date().toISOString(),
|
||||
elapsedMs: 0,
|
||||
};
|
||||
|
||||
let sql: string;
|
||||
try {
|
||||
sql = await this.fingerprintSql();
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
return { ...base, problem: `No se pudo leer el esquema: ${msg}`, elapsedMs: Date.now() - started };
|
||||
}
|
||||
|
||||
let masterOut: string;
|
||||
let replicaOut: string;
|
||||
try {
|
||||
// Sequential, not parallel. Running both at once would have the master
|
||||
// scan under the replica's own read load only sometimes, which makes a
|
||||
// slow run hard to attribute; and the boxes are small enough that two
|
||||
// concurrent full scans is a real memory event on the 946MB replica.
|
||||
masterOut = await this.onMaster(sql);
|
||||
replicaOut = await this.onReplica(sql);
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
return { ...base, problem: `No se pudo comparar: ${msg}`, elapsedMs: Date.now() - started };
|
||||
}
|
||||
|
||||
const master = parseFingerprints(masterOut);
|
||||
const replica = parseFingerprints(replicaOut);
|
||||
|
||||
const tables: TableFingerprint[] = REPLICATED_TABLES.map((table) => {
|
||||
const m = master.get(table);
|
||||
const r = replica.get(table);
|
||||
return {
|
||||
table,
|
||||
masterRows: m?.rows ?? -1,
|
||||
replicaRows: r?.rows ?? -1,
|
||||
masterChecksum: m?.checksum ?? "?",
|
||||
replicaChecksum: r?.checksum ?? "?",
|
||||
// Both sides must have answered. A missing row on either side is a
|
||||
// mismatch, never a pass — `undefined === undefined` would otherwise
|
||||
// report two failed reads as agreement.
|
||||
matches:
|
||||
m !== undefined && r !== undefined && m.rows === r.rows && m.checksum === r.checksum,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
identical: tables.every((t) => t.matches),
|
||||
tables,
|
||||
problem: null,
|
||||
checkedAt: base.checkedAt,
|
||||
elapsedMs: Date.now() - started,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the count+checksum query from the live column list.
|
||||
*
|
||||
* The columns come from `information_schema` on the master rather than being
|
||||
* hardcoded, so the check keeps covering the whole row after a migration adds
|
||||
* one. Reading the schema from the master is safe by construction: if the two
|
||||
* schemas had diverged, replication would already be broken.
|
||||
*/
|
||||
private async fingerprintSql(): Promise<string> {
|
||||
const list = REPLICATED_TABLES.map((t) => `'${t}'`).join(",");
|
||||
const raw = await this.onMaster(
|
||||
"SELECT CONCAT(TABLE_NAME, '\\t', COLUMN_NAME) FROM information_schema.COLUMNS " +
|
||||
`WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (${list}) ` +
|
||||
"ORDER BY TABLE_NAME, ORDINAL_POSITION",
|
||||
["-N"],
|
||||
);
|
||||
|
||||
const cols = new Map<string, string[]>();
|
||||
for (const line of raw.split("\n")) {
|
||||
const [table, column] = line.trim().split("\t");
|
||||
if (!table || !column) continue;
|
||||
cols.set(table, [...(cols.get(table) ?? []), column]);
|
||||
}
|
||||
|
||||
const selects = REPLICATED_TABLES.map((table) => {
|
||||
const columns = cols.get(table);
|
||||
if (!columns?.length) throw new Error(`tabla ${table} sin columnas`);
|
||||
// CONVERT(... USING binary), never CAST(... AS CHAR).
|
||||
//
|
||||
// CAST to CHAR transcodes into the *connection* character set, which is
|
||||
// not the same on the two servers: the mysql client inside the master's
|
||||
// container negotiates latin1, while the replica's negotiates utf8mb4.
|
||||
// Every accented character in a Mexican name, street or note therefore
|
||||
// hashes to different bytes on each side, and the comparison reports a
|
||||
// permanent mismatch on exactly the tables that hold free text — a
|
||||
// verification tool that always cries wolf, which is worse than none.
|
||||
// Comparing the stored bytes sidesteps the session entirely. (Verified
|
||||
// 2026-08-06: with CAST, `customers.name` gave 3344437324815 vs
|
||||
// 3339150372121; with CONVERT both give 3339150372121.)
|
||||
//
|
||||
// 0x1f (unit separator) joins the columns and 0x1e (record separator)
|
||||
// stands in for NULL. Both matter: CONCAT_WS *skips* NULLs rather than
|
||||
// emitting an empty field, so without a placeholder the rows
|
||||
// ('a', NULL, 'b') and ('a', 'b', NULL) produce the same string and a
|
||||
// column-shifting bug would checksum as identical.
|
||||
const expr = columns
|
||||
.map((c) => `IFNULL(CONVERT(\`${c}\` USING binary), 0x1e)`)
|
||||
.join(", 0x1f, ");
|
||||
// SUM, not a running hash: addition is commutative, so the result does not
|
||||
// depend on the order rows come back in. The two servers have no reason to
|
||||
// scan in the same order and are not asked to.
|
||||
return (
|
||||
`SELECT '${table}' AS t, COUNT(*) AS n, ` +
|
||||
`IFNULL(SUM(CRC32(CONCAT_WS(0x1f, ${expr}))), 0) AS c FROM \`${table}\``
|
||||
);
|
||||
});
|
||||
|
||||
return selects.join(" UNION ALL ");
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------ plumbing */
|
||||
|
||||
/**
|
||||
* Run a statement on the replica.
|
||||
*
|
||||
* --ssl is required: the replica sets require_secure_transport=ON.
|
||||
*
|
||||
* --ssl-verify-server-cert=0 is deliberate and is NOT the same trade-off the
|
||||
* website makes. This hop never leaves Tailscale — the replica is reached on
|
||||
* its CGNAT tailnet address and the tailnet ACL admits only this host — so
|
||||
* WireGuard already authenticates the peer. The DreamHost leg crosses the
|
||||
* public internet and therefore pins the CA instead. The client here is
|
||||
* MariaDB's, which rejects our self-signed CA outright unless it is handed the
|
||||
* CA file, which would mean shipping a cert into this image for a link that is
|
||||
* already authenticated.
|
||||
*/
|
||||
private async onReplica(sql: string, extra: string[] = []): Promise<string> {
|
||||
const host = process.env.REPLICA_DB_HOST!;
|
||||
const user = process.env.REPLICA_DB_USER!;
|
||||
const password = process.env.REPLICA_DB_PASS!;
|
||||
const { stdout } = await exec(
|
||||
"mysql",
|
||||
[
|
||||
`--host=${host}`,
|
||||
`--user=${user}`,
|
||||
"--ssl",
|
||||
"--ssl-verify-server-cert=0",
|
||||
"--connect-timeout=5",
|
||||
...extra,
|
||||
"-e",
|
||||
sql,
|
||||
],
|
||||
{ env: { ...process.env, MYSQL_PWD: password }, timeout: VERIFY_TIMEOUT_MS },
|
||||
);
|
||||
return stdout;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a statement on the master, using the application's own DATABASE_URL.
|
||||
*
|
||||
* The app credential is enough here on purpose — everything this class sends
|
||||
* to the master is a SELECT against `information_schema` or a system variable.
|
||||
* Reaching for OPS_DB_ADMIN_* the way OpsService does would hand a monitoring
|
||||
* read path a credential that can also restore a dump.
|
||||
*/
|
||||
private async onMaster(sql: string, extra: string[] = []): Promise<string> {
|
||||
const raw = process.env.DATABASE_URL;
|
||||
if (!raw) throw new Error("DATABASE_URL no está configurada");
|
||||
const u = new URL(raw);
|
||||
const { stdout } = await exec(
|
||||
"mysql",
|
||||
[
|
||||
`--host=${u.hostname}`,
|
||||
`--port=${u.port || "3306"}`,
|
||||
`--user=${decodeURIComponent(u.username)}`,
|
||||
"--connect-timeout=5",
|
||||
...extra,
|
||||
"-N",
|
||||
"-e",
|
||||
sql,
|
||||
u.pathname.replace(/^\//, ""),
|
||||
],
|
||||
{
|
||||
env: { ...process.env, MYSQL_PWD: decodeURIComponent(u.password) },
|
||||
timeout: VERIFY_TIMEOUT_MS,
|
||||
},
|
||||
);
|
||||
return stdout;
|
||||
}
|
||||
}
|
||||
|
||||
/** Full scans on a 1-vCPU replica are not fast; 15s would cut them off. */
|
||||
const VERIFY_TIMEOUT_MS = 120_000;
|
||||
|
||||
/** Parse the `t\tn\tc` rows the fingerprint query emits under `mysql -N`. */
|
||||
function parseFingerprints(raw: string): Map<string, { rows: number; checksum: string }> {
|
||||
const out = new Map<string, { rows: number; checksum: string }>();
|
||||
for (const line of raw.split("\n")) {
|
||||
const [table, n, c] = line.trim().split("\t");
|
||||
if (!table || n === undefined || c === undefined) continue;
|
||||
const rows = Number(n);
|
||||
if (!Number.isFinite(rows)) continue;
|
||||
// The checksum stays a string. Sums of CRC32 over 40k rows exceed 2^53, so
|
||||
// parsing it as a number would round and make distinct tables compare equal.
|
||||
out.set(table, { rows, checksum: c });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Count the transactions in a GTID set.
|
||||
*
|
||||
* Exported for testing. The format is `uuid[:tag]:interval[:interval]...`,
|
||||
* comma-separated, where an interval is `N` or `N-M` inclusive at both ends —
|
||||
* so `1-5` is five transactions, not four.
|
||||
*
|
||||
* MySQL 8.4 added an optional alphanumeric tag between the UUID and the first
|
||||
* interval. It is skipped rather than parsed: any segment that is not a number
|
||||
* or a number range is not an interval, whatever else it may be.
|
||||
*/
|
||||
export function countGtids(set: string): number {
|
||||
if (!set.trim()) return 0;
|
||||
let total = 0;
|
||||
for (const group of set.split(",")) {
|
||||
for (const part of group.trim().split(":").slice(1)) {
|
||||
const m = /^(\d+)(?:-(\d+))?$/.exec(part.trim());
|
||||
if (!m) continue;
|
||||
const from = Number(m[1]);
|
||||
const to = m[2] === undefined ? from : Number(m[2]);
|
||||
if (Number.isFinite(from) && Number.isFinite(to) && to >= from) total += to - from + 1;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/**
|
||||
* Narrow a GTID set to the intervals belonging to one server UUID.
|
||||
*
|
||||
* Exported for testing. Used to isolate the replica's own writes from the
|
||||
* history it replicated, which are interleaved in the same `gtid_executed`.
|
||||
*/
|
||||
export function gtidsForUuid(set: string, uuid: string): string {
|
||||
if (!uuid.trim()) return "";
|
||||
const wanted = uuid.trim().toLowerCase();
|
||||
return set
|
||||
.split(",")
|
||||
.map((g) => g.trim())
|
||||
.filter((g) => g.toLowerCase().startsWith(`${wanted}:`))
|
||||
.join(",");
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import { applyProgress, replicaField } from "./replication.service";
|
||||
import {
|
||||
applyProgress,
|
||||
countGtids,
|
||||
gtidsForUuid,
|
||||
replicaField,
|
||||
} from "./replication.service";
|
||||
|
||||
/**
|
||||
* Verbatim shape of `SHOW REPLICA STATUS\G` from the live replica, trimmed to
|
||||
@@ -178,3 +183,72 @@ describe("applyProgress", () => {
|
||||
expect(applyProgress(positions("binlog.000042", "NULL", "binlog.000042", 400))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Real GTID sets from the live pair, captured 2026-08-06. The replica's own
|
||||
* server UUID (3b103283…) carries the transactions the seed dump load executed
|
||||
* locally; the master's UUID (defc34e2…) carries the replicated history.
|
||||
*/
|
||||
const REPLICA_EXECUTED =
|
||||
"3b103283-8f15-11f1-a52b-020017027b33:1-513," +
|
||||
"defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-525";
|
||||
const REPLICA_UUID = "3b103283-8f15-11f1-a52b-020017027b33";
|
||||
|
||||
describe("countGtids", () => {
|
||||
it("counts an inclusive range at both ends", () => {
|
||||
// 1-5 is five transactions. Off-by-one here understates the gap, which is
|
||||
// the direction that hides a problem.
|
||||
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-5")).toBe(5);
|
||||
});
|
||||
|
||||
it("counts a bare single transaction", () => {
|
||||
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:7")).toBe(1);
|
||||
});
|
||||
|
||||
it("sums several intervals under one UUID", () => {
|
||||
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-5:8:10-12")).toBe(9);
|
||||
});
|
||||
|
||||
it("sums across UUIDs, including the wrapped form MySQL prints", () => {
|
||||
expect(countGtids(REPLICA_EXECUTED)).toBe(513 + 525);
|
||||
// `gtid_executed` comes back wrapped once it holds more than one UUID.
|
||||
expect(countGtids(REPLICA_EXECUTED.replace(",", ",\n"))).toBe(513 + 525);
|
||||
});
|
||||
|
||||
/** An empty subtraction result is the caught-up case and must be zero. */
|
||||
it("returns 0 for an empty or blank set", () => {
|
||||
expect(countGtids("")).toBe(0);
|
||||
expect(countGtids(" \n ")).toBe(0);
|
||||
});
|
||||
|
||||
/**
|
||||
* MySQL 8.4 allows an alphanumeric tag between the UUID and the intervals.
|
||||
* It is not an interval and must not be counted as one.
|
||||
*/
|
||||
it("skips a tag without counting it", () => {
|
||||
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:mytag:1-3")).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("gtidsForUuid", () => {
|
||||
it("isolates the replica's own transactions from the replicated history", () => {
|
||||
expect(countGtids(gtidsForUuid(REPLICA_EXECUTED, REPLICA_UUID))).toBe(513);
|
||||
});
|
||||
|
||||
it("returns nothing for a UUID that is not in the set", () => {
|
||||
expect(gtidsForUuid(REPLICA_EXECUTED, "00000000-0000-0000-0000-000000000000")).toBe("");
|
||||
});
|
||||
|
||||
/**
|
||||
* The colon matters. Without it a UUID prefix would match a longer UUID that
|
||||
* merely starts the same way, and the replica's local writes would be
|
||||
* over-reported.
|
||||
*/
|
||||
it("does not match on a bare prefix", () => {
|
||||
expect(gtidsForUuid(REPLICA_EXECUTED, "3b103283")).toBe("");
|
||||
});
|
||||
|
||||
it("returns nothing when the UUID is blank", () => {
|
||||
expect(gtidsForUuid(REPLICA_EXECUTED, "")).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
listOpsJobs,
|
||||
startOpsJob,
|
||||
uploadIngest,
|
||||
verifyReplication,
|
||||
} from "@/lib/api";
|
||||
import type { UploadProgress } from "@/lib/api";
|
||||
import type {
|
||||
@@ -28,7 +29,9 @@ import type {
|
||||
IngestFile,
|
||||
OpsJob,
|
||||
OpsJobKind,
|
||||
GtidDrift,
|
||||
ReplicationStatus,
|
||||
VerifyResult,
|
||||
} from "@/lib/types";
|
||||
|
||||
const INGEST_MAX_BYTES = 2 * 1024 * 1024 * 1024;
|
||||
@@ -631,6 +634,9 @@ function OpTile({
|
||||
function ReplicationCard() {
|
||||
const [status, setStatus] = useState<ReplicationStatus | null>(null);
|
||||
const [failed, setFailed] = useState(false);
|
||||
const [verify, setVerify] = useState<VerifyResult | null>(null);
|
||||
const [verifying, setVerifying] = useState(false);
|
||||
const [verifyError, setVerifyError] = useState<string | null>(null);
|
||||
|
||||
const load = useCallback(() => {
|
||||
getReplicationStatus()
|
||||
@@ -641,6 +647,17 @@ function ReplicationCard() {
|
||||
.catch(() => setFailed(true));
|
||||
}, []);
|
||||
|
||||
const runVerify = useCallback(() => {
|
||||
setVerifying(true);
|
||||
setVerifyError(null);
|
||||
verifyReplication()
|
||||
.then(setVerify)
|
||||
.catch((e: unknown) =>
|
||||
setVerifyError(e instanceof Error ? e.message : "No se pudo comparar."),
|
||||
)
|
||||
.finally(() => setVerifying(false));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
load();
|
||||
const t = setInterval(load, 30_000);
|
||||
@@ -704,14 +721,144 @@ function ReplicationCard() {
|
||||
value={status.secondsBehind === null ? "sin dato" : `${status.secondsBehind} s`}
|
||||
/>
|
||||
<KV label="Pendiente de aplicar" value={backlogLabel(status.apply)} />
|
||||
<KV label="Diferencia con el maestro" value={driftLabel(status.drift)} />
|
||||
<KV label="Consultado" value={formatDateTime(status.checkedAt)} />
|
||||
</div>
|
||||
|
||||
<ApplyProgressBar apply={status.apply} />
|
||||
|
||||
{/* Only worth showing when it is not zero, and even then as a note rather
|
||||
than a warning: these are the seed load's own transactions, and they
|
||||
are inert until someone tries to promote this box. */}
|
||||
{status.drift !== null && status.drift.localTransactions > 0 && (
|
||||
<p className="inline-form-note" style={{ marginTop: 12 }}>
|
||||
La réplica tiene {status.drift.localTransactions.toLocaleString("es-MX")} transacciones
|
||||
propias (de la carga inicial). No se propagan y no afectan la lectura; sólo importarían
|
||||
si este servidor pasara a ser maestro.
|
||||
</p>
|
||||
)}
|
||||
|
||||
<VerifyPanel
|
||||
result={verify}
|
||||
running={verifying}
|
||||
error={verifyError}
|
||||
onRun={runVerify}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Row-by-row comparison against the master, on demand.
|
||||
*
|
||||
* Separate from the polled fields because it costs a full scan of both servers.
|
||||
* It is the only check here that can catch a row changed on the replica by
|
||||
* something other than replication — the GTID and lag figures would both still
|
||||
* read perfectly healthy in that case.
|
||||
*/
|
||||
function VerifyPanel({
|
||||
result,
|
||||
running,
|
||||
error,
|
||||
onRun,
|
||||
}: {
|
||||
result: VerifyResult | null;
|
||||
running: boolean;
|
||||
error: string | null;
|
||||
onRun: () => void;
|
||||
}) {
|
||||
return (
|
||||
<div style={{ marginTop: 16, borderTop: "1px solid var(--border)", paddingTop: 12 }}>
|
||||
<div className="row-actions" style={{ justifyContent: "space-between" }}>
|
||||
<span className="inline-form-note" style={{ margin: 0 }}>
|
||||
Compara fila por fila las 8 tablas que lee el sitio de clientes. Recorre ambos
|
||||
servidores por completo, así que tarda.
|
||||
</span>
|
||||
<button className="btn btn-ghost" type="button" onClick={onRun} disabled={running}>
|
||||
{running ? "Comparando…" : "Comparar con el maestro"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="state-box state-error" style={{ marginTop: 12 }}>
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{result?.problem && (
|
||||
<div className="state-box state-error" style={{ marginTop: 12 }}>
|
||||
{result.problem}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{result && !result.problem && (
|
||||
<>
|
||||
<p style={{ marginTop: 12, marginBottom: 8 }}>
|
||||
<span className={`badge ${result.identical ? "badge-positive" : "badge-negative"}`}>
|
||||
{result.identical ? "Idénticas" : "Hay diferencias"}
|
||||
</span>{" "}
|
||||
<span className="inline-form-note">
|
||||
{formatDateTime(result.checkedAt)} · {(result.elapsedMs / 1000).toFixed(1)} s
|
||||
</span>
|
||||
</p>
|
||||
<div style={{ overflowX: "auto" }}>
|
||||
<table className="data-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Tabla</th>
|
||||
<th style={{ textAlign: "right" }}>Maestro</th>
|
||||
<th style={{ textAlign: "right" }}>Réplica</th>
|
||||
<th>Estado</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{result.tables.map((t) => (
|
||||
<tr key={t.table}>
|
||||
<td>{t.table}</td>
|
||||
{/* -1 is the sentinel for "that server did not answer for
|
||||
this table", which is not the same as zero rows. */}
|
||||
<td style={{ textAlign: "right" }}>
|
||||
{t.masterRows < 0 ? "—" : t.masterRows.toLocaleString("es-MX")}
|
||||
</td>
|
||||
<td style={{ textAlign: "right" }}>
|
||||
{t.replicaRows < 0 ? "—" : t.replicaRows.toLocaleString("es-MX")}
|
||||
</td>
|
||||
<td>
|
||||
<span className={`badge ${t.matches ? "badge-positive" : "badge-negative"}`}>
|
||||
{t.matches
|
||||
? "igual"
|
||||
: t.masterRows !== t.replicaRows
|
||||
? "difieren en filas"
|
||||
: "difieren en contenido"}
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Transactions the master has executed that the replica has not.
|
||||
*
|
||||
* Rendered as its own line rather than folded into the lag figure because the
|
||||
* two disagree in exactly the case that matters: a disconnected I/O thread
|
||||
* reports 0 seconds of lag (no event has arrived to be late) while this number
|
||||
* climbs.
|
||||
*/
|
||||
function driftLabel(drift: GtidDrift | null): string {
|
||||
// Null means the master could not be reached. Saying "al día" here would be a
|
||||
// lie of the worst kind — it is the reading a broken check produces.
|
||||
if (drift === null) return "sin dato";
|
||||
if (drift.missingTransactions === 0) return "al día";
|
||||
return `${drift.missingTransactions.toLocaleString("es-MX")} transacciones atrás`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bytes the replica has fetched but not yet applied.
|
||||
*
|
||||
|
||||
@@ -60,6 +60,7 @@ import type {
|
||||
OpsJob,
|
||||
OpsJobKind,
|
||||
ReplicationStatus,
|
||||
VerifyResult,
|
||||
IngestFile,
|
||||
BackupFile,
|
||||
PropertyDetail,
|
||||
@@ -964,6 +965,18 @@ export function getReplicationStatus(): Promise<ReplicationStatus> {
|
||||
return apiFetch<ReplicationStatus>("/ops/replication");
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare every customer-visible table against the master, row by row.
|
||||
*
|
||||
* Slow by nature — it is a full scan of both servers — so it is a button, not
|
||||
* part of the poll. Answers the question replication status cannot: GTIDs prove
|
||||
* the replica applied everything the master sent, not that nothing else changed
|
||||
* the rows here.
|
||||
*/
|
||||
export function verifyReplication(): Promise<VerifyResult> {
|
||||
return apiFetch<VerifyResult>("/ops/replication/verify", { method: "POST" });
|
||||
}
|
||||
|
||||
export function listOpsJobs(): Promise<OpsJob[]> {
|
||||
return apiFetch<OpsJob[]>("/ops/jobs");
|
||||
}
|
||||
|
||||
@@ -101,10 +101,52 @@ export interface ReplicationStatus {
|
||||
lastSqlError: string | null;
|
||||
sourceHost: string | null;
|
||||
apply: ApplyProgress | null;
|
||||
drift: GtidDrift | null;
|
||||
problem: string | null;
|
||||
checkedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executed-history gap between master and replica, in transactions.
|
||||
*
|
||||
* The only field on the card that is not self-reported by the replica, and the
|
||||
* only one that catches a silently disconnected I/O thread: with no incoming
|
||||
* events, `secondsBehind` reads 0 because there is nothing to measure staleness
|
||||
* against, so a dead link looks perfectly current. This number grows instead.
|
||||
*
|
||||
* Null when the master could not be reached — "unknown" must not render as
|
||||
* "identical".
|
||||
*/
|
||||
export interface GtidDrift {
|
||||
missingTransactions: number;
|
||||
missingGtidSet: string | null;
|
||||
/**
|
||||
* Transactions written on the replica under its own server UUID, which exist
|
||||
* nowhere on the master. Non-zero is expected — restoring the seed dump
|
||||
* executed its statements locally — and harmless while nothing replicates
|
||||
* from this node.
|
||||
*/
|
||||
localTransactions: number;
|
||||
}
|
||||
|
||||
/** One table compared on both sides of the link. */
|
||||
export interface TableFingerprint {
|
||||
table: string;
|
||||
masterRows: number;
|
||||
replicaRows: number;
|
||||
masterChecksum: string;
|
||||
replicaChecksum: string;
|
||||
matches: boolean;
|
||||
}
|
||||
|
||||
export interface VerifyResult {
|
||||
identical: boolean;
|
||||
tables: TableFingerprint[];
|
||||
problem: string | null;
|
||||
checkedAt: string;
|
||||
elapsedMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Relay-log apply progress, in source binlog bytes.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user