feat(ops): verify the replica against the master, not just its own status

Every field the replication card showed was self-reported by the replica, and
the two most reassuring ones lie in the same failure. Seconds_Behind_Source
reads 0 when the I/O thread is disconnected — with no incoming event there is
nothing to measure staleness against — and Replica_IO_Running only says the
network thread is alive, not that it is receiving.

Two checks that ask the master instead:

- GTID drift, folded into the polled status. GTID_SUBTRACT(master, replica)
  counts transactions the master executed that the replica has not, so a silent
  disconnect shows up as a number that climbs instead of a lag that stays 0.
  It also isolates transactions carried under the replica's OWN server UUID —
  writes that exist nowhere on the master. There are currently 518 of them,
  residue of the seed dump load; inert while log_replica_updates is off, and a
  real divergence the day anyone promotes that box.

- A full row-by-row comparison behind a button, over the eight tables
  my.jorgecuadros.com reads. GTIDs prove the replica applied everything the
  master sent; they say nothing about rows changed here by another route, which
  is the one failure the rest of the card cannot see.

The comparison hashes CONVERT(col USING binary), not CAST(col AS CHAR). CAST
transcodes into the connection character set, and the two servers do not agree
on it: the client inside the master's container negotiates latin1, the replica's
utf8mb4. Every accented character in a Mexican name, street or note then hashes
differently and the tool reports a permanent mismatch on exactly the tables that
hold free text. Caught by building it and running it — customers.name gave
3344437324815 against 3339150372121 under CAST, and 3339150372121 on both under
CONVERT. All eight tables now match byte for byte.

Verify is POST and audited despite reading nothing: it full-scans both servers,
so a prefetch or a refresh must not be able to start one.

Tests cover the GTID interval arithmetic, which is inclusive at both ends and
easy to get wrong by one in the direction that hides a gap.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

apps/web/src/app/operaciones/page.tsx

@@ -20,6 +20,7 @@ import {
   listOpsJobs,
   startOpsJob,
   uploadIngest,
+  verifyReplication,
 } from "@/lib/api";
 import type { UploadProgress } from "@/lib/api";
 import type {
@@ -28,7 +29,9 @@ import type {
   IngestFile,
   OpsJob,
   OpsJobKind,
+  GtidDrift,
   ReplicationStatus,
+  VerifyResult,
 } from "@/lib/types";
 
 const INGEST_MAX_BYTES = 2 * 1024 * 1024 * 1024;
@@ -631,6 +634,9 @@ function OpTile({
 function ReplicationCard() {
   const [status, setStatus] = useState<ReplicationStatus | null>(null);
   const [failed, setFailed] = useState(false);
+  const [verify, setVerify] = useState<VerifyResult | null>(null);
+  const [verifying, setVerifying] = useState(false);
+  const [verifyError, setVerifyError] = useState<string | null>(null);
 
   const load = useCallback(() => {
     getReplicationStatus()
@@ -641,6 +647,17 @@ function ReplicationCard() {
       .catch(() => setFailed(true));
   }, []);
 
+  const runVerify = useCallback(() => {
+    setVerifying(true);
+    setVerifyError(null);
+    verifyReplication()
+      .then(setVerify)
+      .catch((e: unknown) =>
+        setVerifyError(e instanceof Error ? e.message : "No se pudo comparar."),
+      )
+      .finally(() => setVerifying(false));
+  }, []);
+
   useEffect(() => {
     load();
     const t = setInterval(load, 30_000);
@@ -704,14 +721,144 @@ function ReplicationCard() {
           value={status.secondsBehind === null ? "sin dato" : `${status.secondsBehind} s`}
         />
         <KV label="Pendiente de aplicar" value={backlogLabel(status.apply)} />
+        <KV label="Diferencia con el maestro" value={driftLabel(status.drift)} />
         <KV label="Consultado" value={formatDateTime(status.checkedAt)} />
       </div>
 
       <ApplyProgressBar apply={status.apply} />
+
+      {/* Only worth showing when it is not zero, and even then as a note rather
+          than a warning: these are the seed load's own transactions, and they
+          are inert until someone tries to promote this box. */}
+      {status.drift !== null && status.drift.localTransactions > 0 && (
+        <p className="inline-form-note" style={{ marginTop: 12 }}>
+          La réplica tiene {status.drift.localTransactions.toLocaleString("es-MX")} transacciones
+          propias (de la carga inicial). No se propagan y no afectan la lectura; sólo importarían
+          si este servidor pasara a ser maestro.
+        </p>
+      )}
+
+      <VerifyPanel
+        result={verify}
+        running={verifying}
+        error={verifyError}
+        onRun={runVerify}
+      />
     </div>
   );
 }
 
+/**
+ * Row-by-row comparison against the master, on demand.
+ *
+ * Separate from the polled fields because it costs a full scan of both servers.
+ * It is the only check here that can catch a row changed on the replica by
+ * something other than replication — the GTID and lag figures would both still
+ * read perfectly healthy in that case.
+ */
+function VerifyPanel({
+  result,
+  running,
+  error,
+  onRun,
+}: {
+  result: VerifyResult | null;
+  running: boolean;
+  error: string | null;
+  onRun: () => void;
+}) {
+  return (
+    <div style={{ marginTop: 16, borderTop: "1px solid var(--border)", paddingTop: 12 }}>
+      <div className="row-actions" style={{ justifyContent: "space-between" }}>
+        <span className="inline-form-note" style={{ margin: 0 }}>
+          Compara fila por fila las 8 tablas que lee el sitio de clientes. Recorre ambos
+          servidores por completo, así que tarda.
+        </span>
+        <button className="btn btn-ghost" type="button" onClick={onRun} disabled={running}>
+          {running ? "Comparando…" : "Comparar con el maestro"}
+        </button>
+      </div>
+
+      {error && (
+        <div className="state-box state-error" style={{ marginTop: 12 }}>
+          {error}
+        </div>
+      )}
+
+      {result?.problem && (
+        <div className="state-box state-error" style={{ marginTop: 12 }}>
+          {result.problem}
+        </div>
+      )}
+
+      {result && !result.problem && (
+        <>
+          <p style={{ marginTop: 12, marginBottom: 8 }}>
+            <span className={`badge ${result.identical ? "badge-positive" : "badge-negative"}`}>
+              {result.identical ? "Idénticas" : "Hay diferencias"}
+            </span>{" "}
+            <span className="inline-form-note">
+              {formatDateTime(result.checkedAt)} · {(result.elapsedMs / 1000).toFixed(1)} s
+            </span>
+          </p>
+          <div style={{ overflowX: "auto" }}>
+            <table className="data-table">
+              <thead>
+                <tr>
+                  <th>Tabla</th>
+                  <th style={{ textAlign: "right" }}>Maestro</th>
+                  <th style={{ textAlign: "right" }}>Réplica</th>
+                  <th>Estado</th>
+                </tr>
+              </thead>
+              <tbody>
+                {result.tables.map((t) => (
+                  <tr key={t.table}>
+                    <td>{t.table}</td>
+                    {/* -1 is the sentinel for "that server did not answer for
+                        this table", which is not the same as zero rows. */}
+                    <td style={{ textAlign: "right" }}>
+                      {t.masterRows < 0 ? "—" : t.masterRows.toLocaleString("es-MX")}
+                    </td>
+                    <td style={{ textAlign: "right" }}>
+                      {t.replicaRows < 0 ? "—" : t.replicaRows.toLocaleString("es-MX")}
+                    </td>
+                    <td>
+                      <span className={`badge ${t.matches ? "badge-positive" : "badge-negative"}`}>
+                        {t.matches
+                          ? "igual"
+                          : t.masterRows !== t.replicaRows
+                            ? "difieren en filas"
+                            : "difieren en contenido"}
+                      </span>
+                    </td>
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          </div>
+        </>
+      )}
+    </div>
+  );
+}
+
+/**
+ * Transactions the master has executed that the replica has not.
+ *
+ * Rendered as its own line rather than folded into the lag figure because the
+ * two disagree in exactly the case that matters: a disconnected I/O thread
+ * reports 0 seconds of lag (no event has arrived to be late) while this number
+ * climbs.
+ */
+function driftLabel(drift: GtidDrift | null): string {
+  // Null means the master could not be reached. Saying "al día" here would be a
+  // lie of the worst kind — it is the reading a broken check produces.
+  if (drift === null) return "sin dato";
+  if (drift.missingTransactions === 0) return "al día";
+  return `${drift.missingTransactions.toLocaleString("es-MX")} transacciones atrás`;
+}
+
 /**
  * Bytes the replica has fetched but not yet applied.
  *

apps/web/src/lib/api.ts

@@ -60,6 +60,7 @@ import type {
   OpsJob,
   OpsJobKind,
   ReplicationStatus,
+  VerifyResult,
   IngestFile,
   BackupFile,
   PropertyDetail,
@@ -964,6 +965,18 @@ export function getReplicationStatus(): Promise<ReplicationStatus> {
   return apiFetch<ReplicationStatus>("/ops/replication");
 }
 
+/**
+ * Compare every customer-visible table against the master, row by row.
+ *
+ * Slow by nature — it is a full scan of both servers — so it is a button, not
+ * part of the poll. Answers the question replication status cannot: GTIDs prove
+ * the replica applied everything the master sent, not that nothing else changed
+ * the rows here.
+ */
+export function verifyReplication(): Promise<VerifyResult> {
+  return apiFetch<VerifyResult>("/ops/replication/verify", { method: "POST" });
+}
+
 export function listOpsJobs(): Promise<OpsJob[]> {
   return apiFetch<OpsJob[]>("/ops/jobs");
 }

apps/web/src/lib/types.ts

@@ -101,10 +101,52 @@ export interface ReplicationStatus {
   lastSqlError: string | null;
   sourceHost: string | null;
   apply: ApplyProgress | null;
+  drift: GtidDrift | null;
   problem: string | null;
   checkedAt: string;
 }
 
+/**
+ * Executed-history gap between master and replica, in transactions.
+ *
+ * The only field on the card that is not self-reported by the replica, and the
+ * only one that catches a silently disconnected I/O thread: with no incoming
+ * events, `secondsBehind` reads 0 because there is nothing to measure staleness
+ * against, so a dead link looks perfectly current. This number grows instead.
+ *
+ * Null when the master could not be reached — "unknown" must not render as
+ * "identical".
+ */
+export interface GtidDrift {
+  missingTransactions: number;
+  missingGtidSet: string | null;
+  /**
+   * Transactions written on the replica under its own server UUID, which exist
+   * nowhere on the master. Non-zero is expected — restoring the seed dump
+   * executed its statements locally — and harmless while nothing replicates
+   * from this node.
+   */
+  localTransactions: number;
+}
+
+/** One table compared on both sides of the link. */
+export interface TableFingerprint {
+  table: string;
+  masterRows: number;
+  replicaRows: number;
+  masterChecksum: string;
+  replicaChecksum: string;
+  matches: boolean;
+}
+
+export interface VerifyResult {
+  identical: boolean;
+  tables: TableFingerprint[];
+  problem: string | null;
+  checkedAt: string;
+  elapsedMs: number;
+}
+
 /**
  * Relay-log apply progress, in source binlog bytes.
  *