feat(ops): show relay apply progress on the replication card
Seconds_Behind_Source cannot answer "is it moving?". While the SQL thread works through one large transaction the lag counter holds still — often at 0 — even though the replica is not caught up. The relay backlog does move, and it comes out of the SHOW REPLICA STATUS the panel already runs, so this costs no extra query and no connection to the source. Adds applyProgress(), which reads Source_Log_File / Read_Source_Log_Pos vs Relay_Source_Log_File / Exec_Source_Log_Pos and reports the fetched-but-not- applied byte delta plus a percentage. Both positions are source binlog coordinates, so they are only comparable while the two threads are on the same file; across files the delta is meaningless (positions restart at ~4 in each new file) and is reported as null rather than as a huge negative number. The percentage deliberately stops at 99.99 while any backlog remains — binlog positions are large enough that a real backlog of a few KB rounds to 100% and would render a lagging replica as caught up. Not folded into `healthy`: a non-zero backlog is the normal state of a working replica between fetch and apply, so alarming on it would cry wolf. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,45 @@ import { promisify } from "node:util";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
|
||||
/**
|
||||
* How far the SQL thread is behind the I/O thread, in source binlog bytes.
|
||||
*
|
||||
* This is a different question from `secondsBehind`, and it answers the case
|
||||
* that lag hides: while the SQL thread grinds through one huge transaction,
|
||||
* `Seconds_Behind_Source` can sit still or even read 0, but the relay backlog
|
||||
* is plainly shrinking (or not). It costs nothing extra — every field here
|
||||
* comes out of the same `SHOW REPLICA STATUS` the panel already runs.
|
||||
*
|
||||
* Both positions are coordinates in the SOURCE's binlog, so they are only
|
||||
* comparable while both threads are working on the SAME source file. When they
|
||||
* are not, the replica is whole files behind and the byte delta is meaningless
|
||||
* (positions restart at ~4 in each new file), so `backlogBytes` and `percent`
|
||||
* are null and `sameFile` says why.
|
||||
*/
|
||||
export interface ApplyProgress {
|
||||
/** Source binlog file the I/O thread is currently reading. */
|
||||
sourceLogFile: string | null;
|
||||
/** Position in `sourceLogFile` that the I/O thread has fetched up to. */
|
||||
readPos: number;
|
||||
/** Source binlog file the SQL thread is currently applying. */
|
||||
relayLogFile: string | null;
|
||||
/** Position in `relayLogFile` that the SQL thread has applied up to. */
|
||||
execPos: number;
|
||||
/** True while both threads are on the same source file. */
|
||||
sameFile: boolean;
|
||||
/** Fetched-but-not-yet-applied bytes. Null when the files differ. */
|
||||
backlogBytes: number | null;
|
||||
/**
|
||||
* `execPos / readPos` as a percentage, null when the files differ.
|
||||
*
|
||||
* Deliberately never rounded up to 100 while any backlog remains: binlog
|
||||
* positions are large, so a real backlog of a few KB is 99.99% of the file
|
||||
* and would render as "caught up" when it is not. Read `backlogBytes === 0`
|
||||
* for actually caught up.
|
||||
*/
|
||||
percent: number | null;
|
||||
}
|
||||
|
||||
export interface ReplicationStatus {
|
||||
/** false when the replica is not configured for this environment at all. */
|
||||
configured: boolean;
|
||||
@@ -17,6 +56,8 @@ export interface ReplicationStatus {
|
||||
lastIoError: string | null;
|
||||
lastSqlError: string | null;
|
||||
sourceHost: string | null;
|
||||
/** Relay-log apply progress. Null when the status output has no positions. */
|
||||
apply: ApplyProgress | null;
|
||||
/** Human-readable reason when healthy is false. */
|
||||
problem: string | null;
|
||||
checkedAt: string;
|
||||
@@ -57,6 +98,7 @@ export class ReplicationService {
|
||||
lastIoError: null,
|
||||
lastSqlError: null,
|
||||
sourceHost: null,
|
||||
apply: null,
|
||||
problem: null,
|
||||
checkedAt: now,
|
||||
};
|
||||
@@ -142,12 +184,60 @@ export class ReplicationService {
|
||||
lastIoError,
|
||||
lastSqlError,
|
||||
sourceHost: field("Source_Host"),
|
||||
// Reported, never folded into `healthy`: a non-zero backlog is the normal
|
||||
// state of a working replica for the instant between fetch and apply, so
|
||||
// alarming on it would cry wolf. It is here to answer "is it moving?"
|
||||
// when the lag counter is stuck.
|
||||
apply: applyProgress(raw),
|
||||
problem,
|
||||
checkedAt: now,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive relay-apply progress from `SHOW REPLICA STATUS\G` output.
|
||||
*
|
||||
* Exported for testing. Free in query terms — it re-reads four more fields from
|
||||
* the output the caller already has, with no second round trip to the replica
|
||||
* and no connection to the source.
|
||||
*
|
||||
* @returns null when either position is missing or unparseable, which is what
|
||||
* happens on a server that is not a replica at all.
|
||||
*/
|
||||
export function applyProgress(raw: string): ApplyProgress | null {
|
||||
const num = (name: string): number | null => {
|
||||
const v = replicaField(raw, name);
|
||||
if (v === null || v === "NULL") return null;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : null;
|
||||
};
|
||||
|
||||
const readPos = num("Read_Source_Log_Pos");
|
||||
const execPos = num("Exec_Source_Log_Pos");
|
||||
if (readPos === null || execPos === null) return null;
|
||||
|
||||
const sourceLogFile = replicaField(raw, "Source_Log_File");
|
||||
const relayLogFile = replicaField(raw, "Relay_Source_Log_File");
|
||||
const sameFile =
|
||||
sourceLogFile !== null && relayLogFile !== null && sourceLogFile === relayLogFile;
|
||||
|
||||
// Clamped at 0: the SQL thread cannot be ahead of the I/O thread, but the two
|
||||
// fields are sampled independently, so a rotation racing this read can print
|
||||
// a momentarily negative delta. Zero is the honest floor, not a bug.
|
||||
const backlogBytes = sameFile ? Math.max(0, readPos - execPos) : null;
|
||||
|
||||
let percent: number | null = null;
|
||||
if (backlogBytes !== null && readPos > 0) {
|
||||
// Truncate rather than round, and hold short of 100 while bytes remain —
|
||||
// see the doc on ApplyProgress.percent.
|
||||
const p = Math.floor((execPos / readPos) * 10_000) / 100;
|
||||
percent = backlogBytes === 0 ? 100 : Math.min(p, 99.99);
|
||||
}
|
||||
|
||||
return { sourceLogFile, readPos, relayLogFile, execPos, sameFile, backlogBytes, percent };
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one field out of `SHOW REPLICA STATUS\G` output.
|
||||
*
|
||||
|
||||
Reference in New Issue
Block a user