Seconds_Behind_Source cannot answer "is it moving?". While the SQL thread works through one large transaction the lag counter holds still — often at 0 — even though the replica is not caught up. The relay backlog does move, and it comes out of the SHOW REPLICA STATUS the panel already runs, so this costs no extra query and no connection to the source. Adds applyProgress(), which reads Source_Log_File / Read_Source_Log_Pos vs Relay_Source_Log_File / Exec_Source_Log_Pos and reports the fetched-but-not- applied byte delta plus a percentage. Both positions are source binlog coordinates, so they are only comparable while the two threads are on the same file; across files the delta is meaningless (positions restart at ~4 in each new file) and is reported as null rather than as a huge negative number. The percentage deliberately stops at 99.99 while any backlog remains — binlog positions are large enough that a real backlog of a few KB rounds to 100% and would render a lagging replica as caught up. Not folded into `healthy`: a non-zero backlog is the normal state of a working replica between fetch and apply, so alarming on it would cry wolf. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
181 lines
7.2 KiB
TypeScript
181 lines
7.2 KiB
TypeScript
import { applyProgress, replicaField } from "./replication.service";
|
|
|
|
/**
|
|
* Verbatim shape of `SHOW REPLICA STATUS\G` from the live replica, trimmed to
|
|
* the fields the panel reads plus the neighbours that matter.
|
|
*
|
|
* The empty `Last_SQL_Error:` immediately followed by
|
|
* `Replicate_Ignore_Server_Ids:` is the whole point of the fixture — that exact
|
|
* adjacency is what the first implementation misread.
|
|
*/
|
|
const HEALTHY = [
|
|
"*************************** 1. row ***************************",
|
|
" Replica_IO_State: Waiting for source to send event",
|
|
" Source_Host: 100.103.77.46",
|
|
" Source_User: repl",
|
|
" Source_Log_File: binlog.000042",
|
|
" Read_Source_Log_Pos: 194884231",
|
|
" Relay_Source_Log_File: binlog.000042",
|
|
" Exec_Source_Log_Pos: 194884231",
|
|
" Replica_IO_Running: Yes",
|
|
" Replica_SQL_Running: Yes",
|
|
" Replicate_Do_DB: ",
|
|
" Last_Errno: 0",
|
|
" Last_Error: ",
|
|
" Seconds_Behind_Source: 0",
|
|
" Last_IO_Errno: 0",
|
|
" Last_IO_Error: ",
|
|
" Last_SQL_Errno: 0",
|
|
" Last_SQL_Error: ",
|
|
" Replicate_Ignore_Server_Ids: ",
|
|
" Source_Server_Id: 1",
|
|
].join("\n");
|
|
|
|
const BROKEN = [
|
|
" Replica_IO_Running: Yes",
|
|
" Replica_SQL_Running: No",
|
|
" Seconds_Behind_Source: NULL",
|
|
" Last_IO_Error: ",
|
|
" Last_SQL_Error: Could not execute Write_rows event on table jorgecuadros.customers",
|
|
" Replicate_Ignore_Server_Ids: ",
|
|
].join("\n");
|
|
|
|
describe("replicaField", () => {
|
|
it("reads plain values", () => {
|
|
expect(replicaField(HEALTHY, "Replica_IO_Running")).toBe("Yes");
|
|
expect(replicaField(HEALTHY, "Replica_SQL_Running")).toBe("Yes");
|
|
expect(replicaField(HEALTHY, "Source_Host")).toBe("100.103.77.46");
|
|
expect(replicaField(HEALTHY, "Seconds_Behind_Source")).toBe("0");
|
|
});
|
|
|
|
/**
|
|
* The regression this file exists for. `\s` matches newlines in JavaScript,
|
|
* so `^\s*NAME:\s*(.*)$` walks past an empty field's line break and captures
|
|
* the NEXT line — turning a healthy replica into
|
|
* "Error SQL: Replicate_Ignore_Server_Ids:" in the admin panel.
|
|
*/
|
|
it("returns null for an empty field instead of the following line", () => {
|
|
expect(replicaField(HEALTHY, "Last_SQL_Error")).toBeNull();
|
|
expect(replicaField(HEALTHY, "Last_IO_Error")).toBeNull();
|
|
expect(replicaField(HEALTHY, "Last_Error")).toBeNull();
|
|
expect(replicaField(HEALTHY, "Replicate_Do_DB")).toBeNull();
|
|
expect(replicaField(HEALTHY, "Replicate_Ignore_Server_Ids")).toBeNull();
|
|
});
|
|
|
|
it("still reads a real error when there is one", () => {
|
|
expect(replicaField(BROKEN, "Last_SQL_Error")).toBe(
|
|
"Could not execute Write_rows event on table jorgecuadros.customers",
|
|
);
|
|
expect(replicaField(BROKEN, "Replica_SQL_Running")).toBe("No");
|
|
});
|
|
|
|
/** NULL is a distinct state from empty and must survive as the literal. */
|
|
it("preserves the literal NULL that MySQL prints for unknown lag", () => {
|
|
expect(replicaField(BROKEN, "Seconds_Behind_Source")).toBe("NULL");
|
|
});
|
|
|
|
it("returns null for a field that is not present at all", () => {
|
|
expect(replicaField(HEALTHY, "Nonexistent_Field")).toBeNull();
|
|
});
|
|
|
|
/**
|
|
* Field names are matched at the start of a line. Without the line anchor,
|
|
* "Last_Error" would also match inside "Last_SQL_Error" and read the wrong
|
|
* value — the two carry different things and both feed the panel.
|
|
*/
|
|
it("does not match a field name that is a suffix of another", () => {
|
|
const raw = " Last_SQL_Error: boom\n Last_Error: ";
|
|
expect(replicaField(raw, "Last_Error")).toBeNull();
|
|
expect(replicaField(raw, "Last_SQL_Error")).toBe("boom");
|
|
});
|
|
});
|
|
|
|
/** Builds the four position fields the apply-progress reader cares about. */
|
|
function positions(
|
|
sourceFile: string,
|
|
readPos: number | string,
|
|
relayFile: string,
|
|
execPos: number | string,
|
|
): string {
|
|
return [
|
|
` Source_Log_File: ${sourceFile}`,
|
|
` Read_Source_Log_Pos: ${readPos}`,
|
|
` Relay_Source_Log_File: ${relayFile}`,
|
|
` Exec_Source_Log_Pos: ${execPos}`,
|
|
].join("\n");
|
|
}
|
|
|
|
describe("applyProgress", () => {
|
|
it("reports zero backlog and 100% when both positions match", () => {
|
|
const p = applyProgress(HEALTHY)!;
|
|
expect(p.sameFile).toBe(true);
|
|
expect(p.sourceLogFile).toBe("binlog.000042");
|
|
expect(p.readPos).toBe(194884231);
|
|
expect(p.execPos).toBe(194884231);
|
|
expect(p.backlogBytes).toBe(0);
|
|
expect(p.percent).toBe(100);
|
|
});
|
|
|
|
it("reports the byte delta when the SQL thread trails inside one file", () => {
|
|
const p = applyProgress(positions("binlog.000042", 2_000_000, "binlog.000042", 1_500_000))!;
|
|
expect(p.backlogBytes).toBe(500_000);
|
|
expect(p.percent).toBe(75);
|
|
});
|
|
|
|
/**
|
|
* The reason the byte delta exists at all. `Seconds_Behind_Source` holds at 0
|
|
* while the SQL thread is mid-transaction, so the backlog is the only field
|
|
* that moves — and the only one that says the replica is not caught up.
|
|
*/
|
|
it("shows a backlog even when the lag counter reads zero", () => {
|
|
const raw = [
|
|
" Seconds_Behind_Source: 0",
|
|
positions("binlog.000042", 900, "binlog.000042", 400),
|
|
].join("\n");
|
|
expect(replicaField(raw, "Seconds_Behind_Source")).toBe("0");
|
|
expect(applyProgress(raw)!.backlogBytes).toBe(500);
|
|
});
|
|
|
|
/**
|
|
* Positions restart near 4 in every new binlog file, so subtracting across
|
|
* files produces a number that is not a backlog — here it would be a large
|
|
* NEGATIVE one, which would render as "ahead of the source".
|
|
*/
|
|
it("refuses to compare positions across different binlog files", () => {
|
|
const p = applyProgress(positions("binlog.000043", 500, "binlog.000042", 194_000_000))!;
|
|
expect(p.sameFile).toBe(false);
|
|
expect(p.backlogBytes).toBeNull();
|
|
expect(p.percent).toBeNull();
|
|
expect(p.sourceLogFile).toBe("binlog.000043");
|
|
expect(p.relayLogFile).toBe("binlog.000042");
|
|
});
|
|
|
|
/**
|
|
* Percent must not round up to 100 while bytes remain: binlog positions are
|
|
* large, so a genuine backlog is a rounding error away from the whole file
|
|
* and would otherwise render as "caught up" on a replica that is not.
|
|
*/
|
|
it("stops short of 100% while any backlog remains", () => {
|
|
const p = applyProgress(positions("binlog.000042", 194_884_231, "binlog.000042", 194_884_230))!;
|
|
expect(p.backlogBytes).toBe(1);
|
|
expect(p.percent).toBe(99.99);
|
|
});
|
|
|
|
/** Sampled independently, so a rotation racing the read can invert them. */
|
|
it("clamps a momentarily negative delta to zero", () => {
|
|
const p = applyProgress(positions("binlog.000042", 400, "binlog.000042", 500))!;
|
|
expect(p.backlogBytes).toBe(0);
|
|
expect(p.percent).toBe(100);
|
|
});
|
|
|
|
it("returns null when the server is not a replica and prints no positions", () => {
|
|
expect(applyProgress("")).toBeNull();
|
|
expect(applyProgress(BROKEN)).toBeNull();
|
|
});
|
|
|
|
/** A stopped thread makes MySQL print NULL, which is not a position. */
|
|
it("returns null when a position is NULL", () => {
|
|
expect(applyProgress(positions("binlog.000042", "NULL", "binlog.000042", 400))).toBeNull();
|
|
});
|
|
});
|