feat(ops): show relay apply progress on the replication card
Seconds_Behind_Source cannot answer "is it moving?". While the SQL thread works through one large transaction the lag counter holds still — often at 0 — even though the replica is not caught up. The relay backlog does move, and it comes out of the SHOW REPLICA STATUS the panel already runs, so this costs no extra query and no connection to the source. Adds applyProgress(), which reads Source_Log_File / Read_Source_Log_Pos vs Relay_Source_Log_File / Exec_Source_Log_Pos and reports the fetched-but-not- applied byte delta plus a percentage. Both positions are source binlog coordinates, so they are only comparable while the two threads are on the same file; across files the delta is meaningless (positions restart at ~4 in each new file) and is reported as null rather than as a huge negative number. The percentage deliberately stops at 99.99 while any backlog remains — binlog positions are large enough that a real backlog of a few KB rounds to 100% and would render a lagging replica as caught up. Not folded into `healthy`: a non-zero backlog is the normal state of a working replica between fetch and apply, so alarming on it would cry wolf. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,45 @@ import { promisify } from "node:util";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
|
||||
/**
|
||||
* How far the SQL thread is behind the I/O thread, in source binlog bytes.
|
||||
*
|
||||
* This is a different question from `secondsBehind`, and it answers the case
|
||||
* that lag hides: while the SQL thread grinds through one huge transaction,
|
||||
* `Seconds_Behind_Source` can sit still or even read 0, but the relay backlog
|
||||
* is plainly shrinking (or not). It costs nothing extra — every field here
|
||||
* comes out of the same `SHOW REPLICA STATUS` the panel already runs.
|
||||
*
|
||||
* Both positions are coordinates in the SOURCE's binlog, so they are only
|
||||
* comparable while both threads are working on the SAME source file. When they
|
||||
* are not, the replica is whole files behind and the byte delta is meaningless
|
||||
* (positions restart at ~4 in each new file), so `backlogBytes` and `percent`
|
||||
* are null and `sameFile` says why.
|
||||
*/
|
||||
export interface ApplyProgress {
|
||||
/** Source binlog file the I/O thread is currently reading. */
|
||||
sourceLogFile: string | null;
|
||||
/** Position in `sourceLogFile` that the I/O thread has fetched up to. */
|
||||
readPos: number;
|
||||
/** Source binlog file the SQL thread is currently applying. */
|
||||
relayLogFile: string | null;
|
||||
/** Position in `relayLogFile` that the SQL thread has applied up to. */
|
||||
execPos: number;
|
||||
/** True while both threads are on the same source file. */
|
||||
sameFile: boolean;
|
||||
/** Fetched-but-not-yet-applied bytes. Null when the files differ. */
|
||||
backlogBytes: number | null;
|
||||
/**
|
||||
* `execPos / readPos` as a percentage, null when the files differ.
|
||||
*
|
||||
* Deliberately never rounded up to 100 while any backlog remains: binlog
|
||||
* positions are large, so a real backlog of a few KB is 99.99% of the file
|
||||
* and would render as "caught up" when it is not. Read `backlogBytes === 0`
|
||||
* for actually caught up.
|
||||
*/
|
||||
percent: number | null;
|
||||
}
|
||||
|
||||
export interface ReplicationStatus {
|
||||
/** false when the replica is not configured for this environment at all. */
|
||||
configured: boolean;
|
||||
@@ -17,6 +56,8 @@ export interface ReplicationStatus {
|
||||
lastIoError: string | null;
|
||||
lastSqlError: string | null;
|
||||
sourceHost: string | null;
|
||||
/** Relay-log apply progress. Null when the status output has no positions. */
|
||||
apply: ApplyProgress | null;
|
||||
/** Human-readable reason when healthy is false. */
|
||||
problem: string | null;
|
||||
checkedAt: string;
|
||||
@@ -57,6 +98,7 @@ export class ReplicationService {
|
||||
lastIoError: null,
|
||||
lastSqlError: null,
|
||||
sourceHost: null,
|
||||
apply: null,
|
||||
problem: null,
|
||||
checkedAt: now,
|
||||
};
|
||||
@@ -142,12 +184,60 @@ export class ReplicationService {
|
||||
lastIoError,
|
||||
lastSqlError,
|
||||
sourceHost: field("Source_Host"),
|
||||
// Reported, never folded into `healthy`: a non-zero backlog is the normal
|
||||
// state of a working replica for the instant between fetch and apply, so
|
||||
// alarming on it would cry wolf. It is here to answer "is it moving?"
|
||||
// when the lag counter is stuck.
|
||||
apply: applyProgress(raw),
|
||||
problem,
|
||||
checkedAt: now,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive relay-apply progress from `SHOW REPLICA STATUS\G` output.
|
||||
*
|
||||
* Exported for testing. Free in query terms — it re-reads four more fields from
|
||||
* the output the caller already has, with no second round trip to the replica
|
||||
* and no connection to the source.
|
||||
*
|
||||
* @returns null when either position is missing or unparseable, which is what
|
||||
* happens on a server that is not a replica at all.
|
||||
*/
|
||||
export function applyProgress(raw: string): ApplyProgress | null {
|
||||
const num = (name: string): number | null => {
|
||||
const v = replicaField(raw, name);
|
||||
if (v === null || v === "NULL") return null;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
};
|
||||
|
||||
const readPos = num("Read_Source_Log_Pos");
|
||||
const execPos = num("Exec_Source_Log_Pos");
|
||||
if (readPos === null || execPos === null) return null;
|
||||
|
||||
const sourceLogFile = replicaField(raw, "Source_Log_File");
|
||||
const relayLogFile = replicaField(raw, "Relay_Source_Log_File");
|
||||
const sameFile =
|
||||
sourceLogFile !== null && relayLogFile !== null && sourceLogFile === relayLogFile;
|
||||
|
||||
// Clamped at 0: the SQL thread cannot be ahead of the I/O thread, but the two
|
||||
// fields are sampled independently, so a rotation racing this read can print
|
||||
// a momentarily negative delta. Zero is the honest floor, not a bug.
|
||||
const backlogBytes = sameFile ? Math.max(0, readPos - execPos) : null;
|
||||
|
||||
let percent: number | null = null;
|
||||
if (backlogBytes !== null && readPos > 0) {
|
||||
// Truncate rather than round, and hold short of 100 while bytes remain —
|
||||
// see the doc on ApplyProgress.percent.
|
||||
const p = Math.floor((execPos / readPos) * 10_000) / 100;
|
||||
percent = backlogBytes === 0 ? 100 : Math.min(p, 99.99);
|
||||
}
|
||||
|
||||
return { sourceLogFile, readPos, relayLogFile, execPos, sameFile, backlogBytes, percent };
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one field out of `SHOW REPLICA STATUS\G` output.
|
||||
*
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { replicaField } from "./replication.service";
|
||||
import { applyProgress, replicaField } from "./replication.service";
|
||||
|
||||
/**
|
||||
* Verbatim shape of `SHOW REPLICA STATUS\G` from the live replica, trimmed to
|
||||
@@ -13,6 +13,10 @@ const HEALTHY = [
|
||||
" Replica_IO_State: Waiting for source to send event",
|
||||
" Source_Host: 100.103.77.46",
|
||||
" Source_User: repl",
|
||||
" Source_Log_File: binlog.000042",
|
||||
" Read_Source_Log_Pos: 194884231",
|
||||
" Relay_Source_Log_File: binlog.000042",
|
||||
" Exec_Source_Log_Pos: 194884231",
|
||||
" Replica_IO_Running: Yes",
|
||||
" Replica_SQL_Running: Yes",
|
||||
" Replicate_Do_DB: ",
|
||||
@@ -85,3 +89,92 @@ describe("replicaField", () => {
|
||||
expect(replicaField(raw, "Last_SQL_Error")).toBe("boom");
|
||||
});
|
||||
});
|
||||
|
||||
/** Builds the four position fields the apply-progress reader cares about. */
|
||||
function positions(
|
||||
sourceFile: string,
|
||||
readPos: number | string,
|
||||
relayFile: string,
|
||||
execPos: number | string,
|
||||
): string {
|
||||
return [
|
||||
` Source_Log_File: ${sourceFile}`,
|
||||
` Read_Source_Log_Pos: ${readPos}`,
|
||||
` Relay_Source_Log_File: ${relayFile}`,
|
||||
` Exec_Source_Log_Pos: ${execPos}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
describe("applyProgress", () => {
|
||||
it("reports zero backlog and 100% when both positions match", () => {
|
||||
const p = applyProgress(HEALTHY)!;
|
||||
expect(p.sameFile).toBe(true);
|
||||
expect(p.sourceLogFile).toBe("binlog.000042");
|
||||
expect(p.readPos).toBe(194884231);
|
||||
expect(p.execPos).toBe(194884231);
|
||||
expect(p.backlogBytes).toBe(0);
|
||||
expect(p.percent).toBe(100);
|
||||
});
|
||||
|
||||
it("reports the byte delta when the SQL thread trails inside one file", () => {
|
||||
const p = applyProgress(positions("binlog.000042", 2_000_000, "binlog.000042", 1_500_000))!;
|
||||
expect(p.backlogBytes).toBe(500_000);
|
||||
expect(p.percent).toBe(75);
|
||||
});
|
||||
|
||||
/**
|
||||
* The reason the byte delta exists at all. `Seconds_Behind_Source` holds at 0
|
||||
* while the SQL thread is mid-transaction, so the backlog is the only field
|
||||
* that moves — and the only one that says the replica is not caught up.
|
||||
*/
|
||||
it("shows a backlog even when the lag counter reads zero", () => {
|
||||
const raw = [
|
||||
" Seconds_Behind_Source: 0",
|
||||
positions("binlog.000042", 900, "binlog.000042", 400),
|
||||
].join("\n");
|
||||
expect(replicaField(raw, "Seconds_Behind_Source")).toBe("0");
|
||||
expect(applyProgress(raw)!.backlogBytes).toBe(500);
|
||||
});
|
||||
|
||||
/**
|
||||
* Positions restart near 4 in every new binlog file, so subtracting across
|
||||
* files produces a number that is not a backlog — here it would be a large
|
||||
* NEGATIVE one, which would render as "ahead of the source".
|
||||
*/
|
||||
it("refuses to compare positions across different binlog files", () => {
|
||||
const p = applyProgress(positions("binlog.000043", 500, "binlog.000042", 194_000_000))!;
|
||||
expect(p.sameFile).toBe(false);
|
||||
expect(p.backlogBytes).toBeNull();
|
||||
expect(p.percent).toBeNull();
|
||||
expect(p.sourceLogFile).toBe("binlog.000043");
|
||||
expect(p.relayLogFile).toBe("binlog.000042");
|
||||
});
|
||||
|
||||
/**
|
||||
* Percent must not round up to 100 while bytes remain: binlog positions are
|
||||
* large, so a genuine backlog is a rounding error away from the whole file
|
||||
* and would otherwise render as "caught up" on a replica that is not.
|
||||
*/
|
||||
it("stops short of 100% while any backlog remains", () => {
|
||||
const p = applyProgress(positions("binlog.000042", 194_884_231, "binlog.000042", 194_884_230))!;
|
||||
expect(p.backlogBytes).toBe(1);
|
||||
expect(p.percent).toBe(99.99);
|
||||
});
|
||||
|
||||
/** Sampled independently, so a rotation racing the read can invert them. */
|
||||
it("clamps a momentarily negative delta to zero", () => {
|
||||
const p = applyProgress(positions("binlog.000042", 400, "binlog.000042", 500))!;
|
||||
expect(p.backlogBytes).toBe(0);
|
||||
expect(p.percent).toBe(100);
|
||||
});
|
||||
|
||||
it("returns null when the server is not a replica and prints no positions", () => {
|
||||
expect(applyProgress("")).toBeNull();
|
||||
expect(applyProgress(BROKEN)).toBeNull();
|
||||
});
|
||||
|
||||
/** A stopped thread makes MySQL print NULL, which is not a position. */
|
||||
it("returns null when a position is NULL", () => {
|
||||
expect(applyProgress(positions("binlog.000042", "NULL", "binlog.000042", 400))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
} from "@/lib/api";
|
||||
import type { UploadProgress } from "@/lib/api";
|
||||
import type {
|
||||
ApplyProgress,
|
||||
BackupFile,
|
||||
IngestFile,
|
||||
OpsJob,
|
||||
@@ -685,8 +686,58 @@ function ReplicationCard() {
|
||||
label="Retraso"
|
||||
value={status.secondsBehind === null ? "sin dato" : `${status.secondsBehind} s`}
|
||||
/>
|
||||
<KV label="Pendiente de aplicar" value={backlogLabel(status.apply)} />
|
||||
<KV label="Consultado" value={formatDateTime(status.checkedAt)} />
|
||||
</div>
|
||||
|
||||
<ApplyProgressBar apply={status.apply} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bytes the replica has fetched but not yet applied.
|
||||
*
|
||||
* Kept separate from the lag figure because it answers a question the lag
|
||||
* cannot: while the SQL thread chews through one big transaction, the seconds
|
||||
* counter can hold still, but this number visibly falls.
|
||||
*/
|
||||
function backlogLabel(apply: ApplyProgress | null): string {
|
||||
if (!apply) return "sin dato";
|
||||
// Different source binlog files means the replica is whole files behind and
|
||||
// the byte delta is not a delta at all — positions restart in each new file.
|
||||
if (!apply.sameFile) return "más de un archivo de binlog";
|
||||
if (apply.backlogBytes === 0) return "al día";
|
||||
return formatBytes(apply.backlogBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Applied-vs-fetched bar. Rendered only when both threads are on the same
|
||||
* source binlog file, because that is the only case where the percentage is
|
||||
* arithmetic rather than a guess.
|
||||
*/
|
||||
function ApplyProgressBar({ apply }: { apply: ApplyProgress | null }) {
|
||||
if (!apply || !apply.sameFile || apply.percent === null) return null;
|
||||
|
||||
return (
|
||||
<div className="upload-progress" style={{ marginTop: 12 }}>
|
||||
<div
|
||||
className="progress-track"
|
||||
role="progressbar"
|
||||
aria-valuenow={apply.percent}
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-label="Eventos aplicados de los recibidos"
|
||||
>
|
||||
<div className="progress-fill" style={{ width: `${apply.percent}%` }} />
|
||||
</div>
|
||||
<div className="upload-progress-stats mono">
|
||||
<span>{apply.percent}% aplicado</span>
|
||||
<span>
|
||||
{apply.sourceLogFile} · {apply.execPos.toLocaleString("es-MX")} /{" "}
|
||||
{apply.readPos.toLocaleString("es-MX")}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -99,10 +99,32 @@ export interface ReplicationStatus {
|
||||
lastIoError: string | null;
|
||||
lastSqlError: string | null;
|
||||
sourceHost: string | null;
|
||||
apply: ApplyProgress | null;
|
||||
problem: string | null;
|
||||
checkedAt: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Relay-log apply progress, in source binlog bytes.
|
||||
*
|
||||
* Answers "is it moving?" when `secondsBehind` cannot: the lag counter sits
|
||||
* still while the SQL thread works through one large transaction, but the
|
||||
* backlog visibly shrinks. `backlogBytes === 0` is the only reading that means
|
||||
* caught up — `percent` deliberately stops at 99.99 while bytes remain.
|
||||
*
|
||||
* Null fields when the two threads are on different source binlog files
|
||||
* (`sameFile === false`), because the positions are then not comparable.
|
||||
*/
|
||||
export interface ApplyProgress {
|
||||
sourceLogFile: string | null;
|
||||
readPos: number;
|
||||
relayLogFile: string | null;
|
||||
execPos: number;
|
||||
sameFile: boolean;
|
||||
backlogBytes: number | null;
|
||||
percent: number | null;
|
||||
}
|
||||
|
||||
/** One of the four legacy Access files expected in the ingest folder. */
|
||||
export interface IngestFile {
|
||||
name: string;
|
||||
|
||||
Reference in New Issue
Block a user