Files
jorgecuadros-platform/apps/api/src/ops/replication.spec.ts
T
rmancinasandClaude Opus 5 2169ffa78d
Build and Push Images / Build jorgecuadros-web (push) Successful in 1m51s
Build and Push Images / Build jorgecuadros-api (push) Successful in 2m9s
feat(ops): verify the replica against the master, not just its own status
Every field the replication card showed was self-reported by the replica, and
the two most reassuring ones lie in the same failure. Seconds_Behind_Source
reads 0 when the I/O thread is disconnected — with no incoming event there is
nothing to measure staleness against — and Replica_IO_Running only says the
network thread is alive, not that it is receiving.

Two checks that ask the master instead:

- GTID drift, folded into the polled status. GTID_SUBTRACT(master, replica)
  counts transactions the master executed that the replica has not, so a silent
  disconnect shows up as a number that climbs instead of a lag that stays 0.
  It also isolates transactions carried under the replica's OWN server UUID —
  writes that exist nowhere on the master. There are currently 518 of them,
  residue of the seed dump load; inert while log_replica_updates is off, and a
  real divergence the day anyone promotes that box.

- A full row-by-row comparison behind a button, over the eight tables
  my.jorgecuadros.com reads. GTIDs prove the replica applied everything the
  master sent; they say nothing about rows changed here by another route, which
  is the one failure the rest of the card cannot see.

The comparison hashes CONVERT(col USING binary), not CAST(col AS CHAR). CAST
transcodes into the connection character set, and the two servers do not agree
on it: the client inside the master's container negotiates latin1, the replica's
utf8mb4. Every accented character in a Mexican name, street or note then hashes
differently and the tool reports a permanent mismatch on exactly the tables that
hold free text. Caught by building it and running it — customers.name gave
3344437324815 against 3339150372121 under CAST, and 3339150372121 on both under
CONVERT. All eight tables now match byte for byte.

Verify is POST and audited despite reading nothing: it full-scans both servers,
so a prefetch or a refresh must not be able to start one.

Tests cover the GTID interval arithmetic, which is inclusive at both ends and
easy to get wrong by one in the direction that hides a gap.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-06 21:38:22 -07:00

255 lines
9.8 KiB
TypeScript

import {
applyProgress,
countGtids,
gtidsForUuid,
replicaField,
} from "./replication.service";
/**
* Verbatim shape of `SHOW REPLICA STATUS\G` from the live replica, trimmed to
* the fields the panel reads plus the neighbours that matter.
*
* The empty `Last_SQL_Error:` immediately followed by
* `Replicate_Ignore_Server_Ids:` is the whole point of the fixture — that exact
* adjacency is what the first implementation misread.
*/
const HEALTHY = [
"*************************** 1. row ***************************",
" Replica_IO_State: Waiting for source to send event",
" Source_Host: 100.103.77.46",
" Source_User: repl",
" Source_Log_File: binlog.000042",
" Read_Source_Log_Pos: 194884231",
" Relay_Source_Log_File: binlog.000042",
" Exec_Source_Log_Pos: 194884231",
" Replica_IO_Running: Yes",
" Replica_SQL_Running: Yes",
" Replicate_Do_DB: ",
" Last_Errno: 0",
" Last_Error: ",
" Seconds_Behind_Source: 0",
" Last_IO_Errno: 0",
" Last_IO_Error: ",
" Last_SQL_Errno: 0",
" Last_SQL_Error: ",
" Replicate_Ignore_Server_Ids: ",
" Source_Server_Id: 1",
].join("\n");
const BROKEN = [
" Replica_IO_Running: Yes",
" Replica_SQL_Running: No",
" Seconds_Behind_Source: NULL",
" Last_IO_Error: ",
" Last_SQL_Error: Could not execute Write_rows event on table jorgecuadros.customers",
" Replicate_Ignore_Server_Ids: ",
].join("\n");
describe("replicaField", () => {
it("reads plain values", () => {
expect(replicaField(HEALTHY, "Replica_IO_Running")).toBe("Yes");
expect(replicaField(HEALTHY, "Replica_SQL_Running")).toBe("Yes");
expect(replicaField(HEALTHY, "Source_Host")).toBe("100.103.77.46");
expect(replicaField(HEALTHY, "Seconds_Behind_Source")).toBe("0");
});
/**
* The regression this file exists for. `\s` matches newlines in JavaScript,
* so `^\s*NAME:\s*(.*)$` walks past an empty field's line break and captures
* the NEXT line — turning a healthy replica into
* "Error SQL: Replicate_Ignore_Server_Ids:" in the admin panel.
*/
it("returns null for an empty field instead of the following line", () => {
expect(replicaField(HEALTHY, "Last_SQL_Error")).toBeNull();
expect(replicaField(HEALTHY, "Last_IO_Error")).toBeNull();
expect(replicaField(HEALTHY, "Last_Error")).toBeNull();
expect(replicaField(HEALTHY, "Replicate_Do_DB")).toBeNull();
expect(replicaField(HEALTHY, "Replicate_Ignore_Server_Ids")).toBeNull();
});
it("still reads a real error when there is one", () => {
expect(replicaField(BROKEN, "Last_SQL_Error")).toBe(
"Could not execute Write_rows event on table jorgecuadros.customers",
);
expect(replicaField(BROKEN, "Replica_SQL_Running")).toBe("No");
});
/** NULL is a distinct state from empty and must survive as the literal. */
it("preserves the literal NULL that MySQL prints for unknown lag", () => {
expect(replicaField(BROKEN, "Seconds_Behind_Source")).toBe("NULL");
});
it("returns null for a field that is not present at all", () => {
expect(replicaField(HEALTHY, "Nonexistent_Field")).toBeNull();
});
/**
* Field names are matched at the start of a line. Without the line anchor,
* "Last_Error" would also match inside "Last_SQL_Error" and read the wrong
* value — the two carry different things and both feed the panel.
*/
it("does not match a field name that is a suffix of another", () => {
const raw = " Last_SQL_Error: boom\n Last_Error: ";
expect(replicaField(raw, "Last_Error")).toBeNull();
expect(replicaField(raw, "Last_SQL_Error")).toBe("boom");
});
});
/** Builds the four position fields the apply-progress reader cares about. */
function positions(
sourceFile: string,
readPos: number | string,
relayFile: string,
execPos: number | string,
): string {
return [
` Source_Log_File: ${sourceFile}`,
` Read_Source_Log_Pos: ${readPos}`,
` Relay_Source_Log_File: ${relayFile}`,
` Exec_Source_Log_Pos: ${execPos}`,
].join("\n");
}
describe("applyProgress", () => {
it("reports zero backlog and 100% when both positions match", () => {
const p = applyProgress(HEALTHY)!;
expect(p.sameFile).toBe(true);
expect(p.sourceLogFile).toBe("binlog.000042");
expect(p.readPos).toBe(194884231);
expect(p.execPos).toBe(194884231);
expect(p.backlogBytes).toBe(0);
expect(p.percent).toBe(100);
});
it("reports the byte delta when the SQL thread trails inside one file", () => {
const p = applyProgress(positions("binlog.000042", 2_000_000, "binlog.000042", 1_500_000))!;
expect(p.backlogBytes).toBe(500_000);
expect(p.percent).toBe(75);
});
/**
* The reason the byte delta exists at all. `Seconds_Behind_Source` holds at 0
* while the SQL thread is mid-transaction, so the backlog is the only field
* that moves — and the only one that says the replica is not caught up.
*/
it("shows a backlog even when the lag counter reads zero", () => {
const raw = [
" Seconds_Behind_Source: 0",
positions("binlog.000042", 900, "binlog.000042", 400),
].join("\n");
expect(replicaField(raw, "Seconds_Behind_Source")).toBe("0");
expect(applyProgress(raw)!.backlogBytes).toBe(500);
});
/**
* Positions restart near 4 in every new binlog file, so subtracting across
* files produces a number that is not a backlog — here it would be a large
* NEGATIVE one, which would render as "ahead of the source".
*/
it("refuses to compare positions across different binlog files", () => {
const p = applyProgress(positions("binlog.000043", 500, "binlog.000042", 194_000_000))!;
expect(p.sameFile).toBe(false);
expect(p.backlogBytes).toBeNull();
expect(p.percent).toBeNull();
expect(p.sourceLogFile).toBe("binlog.000043");
expect(p.relayLogFile).toBe("binlog.000042");
});
/**
* Percent must not round up to 100 while bytes remain: binlog positions are
* large, so a genuine backlog is a rounding error away from the whole file
* and would otherwise render as "caught up" on a replica that is not.
*/
it("stops short of 100% while any backlog remains", () => {
const p = applyProgress(positions("binlog.000042", 194_884_231, "binlog.000042", 194_884_230))!;
expect(p.backlogBytes).toBe(1);
expect(p.percent).toBe(99.99);
});
/** Sampled independently, so a rotation racing the read can invert them. */
it("clamps a momentarily negative delta to zero", () => {
const p = applyProgress(positions("binlog.000042", 400, "binlog.000042", 500))!;
expect(p.backlogBytes).toBe(0);
expect(p.percent).toBe(100);
});
it("returns null when the server is not a replica and prints no positions", () => {
expect(applyProgress("")).toBeNull();
expect(applyProgress(BROKEN)).toBeNull();
});
/** A stopped thread makes MySQL print NULL, which is not a position. */
it("returns null when a position is NULL", () => {
expect(applyProgress(positions("binlog.000042", "NULL", "binlog.000042", 400))).toBeNull();
});
});
/**
* Real GTID sets from the live pair, captured 2026-08-06. The replica's own
* server UUID (3b103283…) carries the transactions the seed dump load executed
* locally; the master's UUID (defc34e2…) carries the replicated history.
*/
const REPLICA_EXECUTED =
"3b103283-8f15-11f1-a52b-020017027b33:1-513," +
"defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-525";
const REPLICA_UUID = "3b103283-8f15-11f1-a52b-020017027b33";
describe("countGtids", () => {
it("counts an inclusive range at both ends", () => {
// 1-5 is five transactions. Off-by-one here understates the gap, which is
// the direction that hides a problem.
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-5")).toBe(5);
});
it("counts a bare single transaction", () => {
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:7")).toBe(1);
});
it("sums several intervals under one UUID", () => {
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-5:8:10-12")).toBe(9);
});
it("sums across UUIDs, including the wrapped form MySQL prints", () => {
expect(countGtids(REPLICA_EXECUTED)).toBe(513 + 525);
// `gtid_executed` comes back wrapped once it holds more than one UUID.
expect(countGtids(REPLICA_EXECUTED.replace(",", ",\n"))).toBe(513 + 525);
});
/** An empty subtraction result is the caught-up case and must be zero. */
it("returns 0 for an empty or blank set", () => {
expect(countGtids("")).toBe(0);
expect(countGtids(" \n ")).toBe(0);
});
/**
* MySQL 8.4 allows an alphanumeric tag between the UUID and the intervals.
* It is not an interval and must not be counted as one.
*/
it("skips a tag without counting it", () => {
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:mytag:1-3")).toBe(3);
});
});
describe("gtidsForUuid", () => {
it("isolates the replica's own transactions from the replicated history", () => {
expect(countGtids(gtidsForUuid(REPLICA_EXECUTED, REPLICA_UUID))).toBe(513);
});
it("returns nothing for a UUID that is not in the set", () => {
expect(gtidsForUuid(REPLICA_EXECUTED, "00000000-0000-0000-0000-000000000000")).toBe("");
});
/**
* The colon matters. Without it a UUID prefix would match a longer UUID that
* merely starts the same way, and the replica's local writes would be
* over-reported.
*/
it("does not match on a bare prefix", () => {
expect(gtidsForUuid(REPLICA_EXECUTED, "3b103283")).toBe("");
});
it("returns nothing when the UUID is blank", () => {
expect(gtidsForUuid(REPLICA_EXECUTED, "")).toBe("");
});
});