diff --git a/.gitea/workflows/deploy-galactus.yml b/.gitea/workflows/deploy-galactus.yml index 839bf84..907e194 100644 --- a/.gitea/workflows/deploy-galactus.yml +++ b/.gitea/workflows/deploy-galactus.yml @@ -295,6 +295,9 @@ jobs: "SESSION_COOKIE_SECURE": "false", "OPS_DB_ADMIN_USER": "root", "OPS_DB_ADMIN_PASSWORD": "${{ secrets.MYSQL_ROOT_PASSWORD }}", + "REPLICA_DB_HOST": "${{ secrets.REPLICA_DB_HOST }}", + "REPLICA_DB_USER": "${{ secrets.REPLICA_DB_USER }}", + "REPLICA_DB_PASS": "${{ secrets.REPLICA_DB_PASS }}", "MINIO_ROOT_USER": "${{ secrets.MINIO_ROOT_USER }}", "MINIO_ROOT_PASSWORD": "${{ secrets.MINIO_ROOT_PASSWORD }}", "SES_REGION": "${{ secrets.SES_REGION }}", diff --git a/apps/api/src/ops/ops.controller.ts b/apps/api/src/ops/ops.controller.ts index 0d37b39..a73f07a 100644 --- a/apps/api/src/ops/ops.controller.ts +++ b/apps/api/src/ops/ops.controller.ts @@ -19,6 +19,7 @@ import { AbilityGuard } from "../auth/ability.guard"; import { RequireAbility } from "../auth/require-ability.decorator"; import { AuditService } from "../common/audit.service"; import { OpsService } from "./ops.service"; +import { ReplicationService } from "./replication.service"; import { StartJobDto } from "./start-job.dto"; /** Every route is ADMIN-only (ability "db:manage"). */ @@ -28,6 +29,7 @@ import { StartJobDto } from "./start-job.dto"; export class OpsController { constructor( private readonly ops: OpsService, + private readonly replication: ReplicationService, private readonly audit: AuditService, ) {} @@ -96,6 +98,12 @@ export class OpsController { /* --------------------------------------------------------------- jobs */ + /** Health of the my.jorgecuadros.com read replica. Read-only, no audit entry. */ + @Get("replication") + replicationStatus() { + return this.replication.status(); + } + @Get("jobs") listJobs() { return this.ops.listJobs(); diff --git a/apps/api/src/ops/ops.module.ts b/apps/api/src/ops/ops.module.ts index 05824e1..5d6ed49 100644 --- a/apps/api/src/ops/ops.module.ts +++ b/apps/api/src/ops/ops.module.ts @@ -1,9 +1,10 @@ import { Module } from "@nestjs/common"; import { OpsController } from "./ops.controller"; import { OpsService } from "./ops.service"; +import { ReplicationService } from "./replication.service"; @Module({ controllers: [OpsController], - providers: [OpsService], + providers: [OpsService, ReplicationService], }) export class OpsModule {} diff --git a/apps/api/src/ops/replication.service.ts b/apps/api/src/ops/replication.service.ts new file mode 100644 index 0000000..c872b06 --- /dev/null +++ b/apps/api/src/ops/replication.service.ts @@ -0,0 +1,153 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const exec = promisify(execFile); + +export interface ReplicationStatus { + /** false when the replica is not configured for this environment at all. */ + configured: boolean; + /** true only when both threads run, no error is set, and lag is within bounds. */ + healthy: boolean; + host: string | null; + ioRunning: string | null; + sqlRunning: string | null; + /** null when MySQL reports NULL, which it does whenever a thread is down. */ + secondsBehind: number | null; + lastIoError: string | null; + lastSqlError: string | null; + sourceHost: string | null; + /** Human-readable reason when healthy is false. */ + problem: string | null; + checkedAt: string; +} + +/** + * Reports whether the my.jorgecuadros.com read replica is still replicating. + * + * The replica is what the public site reads once the platformDataSource flag is + * on, and a replica that has silently stopped applying serves stale balances + * rather than erroring — the failure is invisible from the site itself, which is + * why it needs a panel. + * + * Shells out to the mysql client for the same reason the rest of OpsService + * does: there is no MySQL driver in this API's dependencies, and the image + * already ships one. + */ +@Injectable() +export class ReplicationService { + private readonly logger = new Logger(ReplicationService.name); + + /** Lag above this many seconds is reported as unhealthy. */ + private readonly maxLagSeconds = Number(process.env.REPLICA_MAX_LAG ?? 60); + + async status(): Promise { + const host = process.env.REPLICA_DB_HOST; + const user = process.env.REPLICA_DB_USER; + const password = process.env.REPLICA_DB_PASS; + const now = new Date().toISOString(); + + const empty: ReplicationStatus = { + configured: false, + healthy: false, + host: host ?? null, + ioRunning: null, + sqlRunning: null, + secondsBehind: null, + lastIoError: null, + lastSqlError: null, + sourceHost: null, + problem: null, + checkedAt: now, + }; + + if (!host || !user || !password) { + return { ...empty, problem: "REPLICA_DB_* no configuradas" }; + } + + let raw: string; + try { + // --ssl is required: the replica sets require_secure_transport=ON. + // + // --ssl-verify-server-cert=0 is deliberate and is NOT the same trade-off + // the website makes. This hop never leaves Tailscale — the replica is + // reached on its CGNAT tailnet address and its firewall admits only this + // host — so WireGuard already authenticates the peer. The DreamHost leg + // crosses the public internet and therefore pins the CA instead. The + // client here is MariaDB's, which rejects our self-signed CA outright + // unless it is handed the CA file, which would mean shipping a cert into + // this image for a link that is already authenticated. + const { stdout } = await exec( + "mysql", + [ + `--host=${host}`, + `--user=${user}`, + "--ssl", + "--ssl-verify-server-cert=0", + "--connect-timeout=5", + "-e", + "SHOW REPLICA STATUS\\G", + ], + { + env: { ...process.env, MYSQL_PWD: password }, + timeout: 15_000, + }, + ); + raw = stdout; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + this.logger.warn(`no se pudo consultar la réplica: ${msg}`); + return { ...empty, configured: true, problem: `No se pudo conectar: ${msg}` }; + } + + const field = (name: string): string | null => { + const m = raw.match(new RegExp(`^\\s*${name}:\\s*(.*)$`, "m")); + const v = m?.[1]?.trim(); + return v === undefined || v === "" ? null : v; + }; + + // An empty result set means the server is not configured as a replica at + // all — distinct from "configured but broken", and worth saying plainly. + if (!raw.includes("Replica_IO_Running")) { + return { + ...empty, + configured: true, + problem: "El servidor no está configurado como réplica", + }; + } + + const ioRunning = field("Replica_IO_Running"); + const sqlRunning = field("Replica_SQL_Running"); + const lagRaw = field("Seconds_Behind_Source"); + const secondsBehind = + lagRaw === null || lagRaw === "NULL" ? null : Number(lagRaw); + const lastIoError = field("Last_IO_Error"); + const lastSqlError = field("Last_SQL_Error"); + + // Order matters: report the most specific cause first. Checking lag before + // the threads would blame "sin dato de retraso" for what is really a + // stopped thread, because MySQL reports NULL lag whenever either is down. + let problem: string | null = null; + if (ioRunning !== "Yes") problem = "El hilo de E/S no está corriendo"; + else if (sqlRunning !== "Yes") problem = "El hilo SQL no está corriendo"; + else if (lastSqlError) problem = `Error SQL: ${lastSqlError}`; + else if (lastIoError) problem = `Error de E/S: ${lastIoError}`; + else if (secondsBehind === null) problem = "Sin dato de retraso"; + else if (secondsBehind > this.maxLagSeconds) + problem = `Retraso de ${secondsBehind}s (máximo ${this.maxLagSeconds}s)`; + + return { + configured: true, + healthy: problem === null, + host, + ioRunning, + sqlRunning, + secondsBehind, + lastIoError, + lastSqlError, + sourceHost: field("Source_Host"), + problem, + checkedAt: now, + }; + } +} diff --git a/apps/web/src/app/operaciones/page.tsx b/apps/web/src/app/operaciones/page.tsx index abeece9..5489551 100644 --- a/apps/web/src/app/operaciones/page.tsx +++ b/apps/web/src/app/operaciones/page.tsx @@ -14,6 +14,7 @@ import { deleteBackup, deleteIngest, getOpsJob, + getReplicationStatus, listBackups, listIngest, listOpsJobs, @@ -26,6 +27,7 @@ import type { IngestFile, OpsJob, OpsJobKind, + ReplicationStatus, } from "@/lib/types"; const INGEST_MAX_BYTES = 2 * 1024 * 1024 * 1024; @@ -227,6 +229,8 @@ function Operaciones() { )} + + {/* Ingest folder */}

Carpeta de ingesta

@@ -596,3 +600,102 @@ function OpTile({
); } + +/** + * Health of the read replica behind my.jorgecuadros.com. + * + * Worth a panel because the failure mode is silent: a replica whose SQL thread + * has stopped keeps answering queries, just with data frozen at the moment it + * stopped. Nothing on the customer site looks wrong — the balances are simply + * out of date — so without this the only signal is a customer complaining. + */ +function ReplicationCard() { + const [status, setStatus] = useState(null); + const [failed, setFailed] = useState(false); + + const load = useCallback(() => { + getReplicationStatus() + .then((s) => { + setStatus(s); + setFailed(false); + }) + .catch(() => setFailed(true)); + }, []); + + useEffect(() => { + load(); + const t = setInterval(load, 30_000); + return () => clearInterval(t); + }, [load]); + + // Not configured is the normal state in dev and before cutover, so it is a + // quiet note rather than an alarm — showing red here would train people to + // ignore the card. + if (failed || (status && !status.configured)) { + return ( +
+

Réplica del sitio de clientes

+

+ {failed + ? "No se pudo consultar el estado de la réplica." + : "No configurada en este entorno."} +

+
+ ); + } + + if (!status) { + return ( +
+

Réplica del sitio de clientes

+

Consultando…

+
+ ); + } + + return ( +
+
+

+ Réplica del sitio de clientes{" "} + + {status.healthy ? "Replicando" : "Detenida"} + +

+ +
+ + {status.problem && ( +
+ {status.problem} +
+ )} + +
+ + + + + {/* Never render a null lag as "0 s": MySQL reports NULL whenever a + thread is down, so the honest word is "unknown", not "up to date". */} + + +
+
+ ); +} + +/** Matches the KV in the clientes/polizas/servicios detail pages. */ +function KV({ label, value }: { label: string; value: string | null | undefined }) { + return ( +
+
{label}
+
{value || "—"}
+
+ ); +} diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index ac9d241..7d8ee25 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -59,6 +59,7 @@ import type { LookupsResponse, OpsJob, OpsJobKind, + ReplicationStatus, IngestFile, BackupFile, PropertyDetail, @@ -939,6 +940,16 @@ export function deleteBackup(name: string): Promise { return apiFetch(`/ops/backups/${encodeURIComponent(name)}`, { method: "DELETE" }); } +/** + * Health of the read replica my.jorgecuadros.com serves customers from. + * + * A stopped replica does not error — it answers with stale balances — so this + * is the only place the failure is visible. + */ +export function getReplicationStatus(): Promise { + return apiFetch("/ops/replication"); +} + export function listOpsJobs(): Promise { return apiFetch("/ops/jobs"); } diff --git a/apps/web/src/lib/types.ts b/apps/web/src/lib/types.ts index 17484ff..57f737f 100644 --- a/apps/web/src/lib/types.ts +++ b/apps/web/src/lib/types.ts @@ -72,6 +72,27 @@ export interface OpsJob { finishedAt: string | null; } +/** + * Health of the MySQL read replica that my.jorgecuadros.com queries. + * + * `secondsBehind` is null whenever MySQL reports NULL, which it does when + * EITHER thread is down — so null means "unknown", never "up to date". Read + * `healthy`/`problem` rather than inferring health from the lag. + */ +export interface ReplicationStatus { + configured: boolean; + healthy: boolean; + host: string | null; + ioRunning: string | null; + sqlRunning: string | null; + secondsBehind: number | null; + lastIoError: string | null; + lastSqlError: string | null; + sourceHost: string | null; + problem: string | null; + checkedAt: string; +} + /** One of the four legacy Access files expected in the ingest folder. */ export interface IngestFile { name: string; diff --git a/deploy/galactus/jorgecuadros-app.compose.yml b/deploy/galactus/jorgecuadros-app.compose.yml index 92dafaa..3a5c3a1 100644 --- a/deploy/galactus/jorgecuadros-app.compose.yml +++ b/deploy/galactus/jorgecuadros-app.compose.yml @@ -69,6 +69,14 @@ services: # apps/api/src/ops/ops.service.ts. OPS_DB_ADMIN_USER: ${OPS_DB_ADMIN_USER:-root} OPS_DB_ADMIN_PASSWORD: ${OPS_DB_ADMIN_PASSWORD:?OPS_DB_ADMIN_PASSWORD must be set} + # Read-only replica that my.jorgecuadros.com serves customers from. Used + # ONLY to report health on the Operaciones screen — the account holds + # REPLICATION CLIENT and nothing else, so it cannot read a single row. + # Unset is a supported state: the panel then says "no configurada" + # instead of erroring, which is correct before cutover and in dev. + REPLICA_DB_HOST: ${REPLICA_DB_HOST:-} + REPLICA_DB_USER: ${REPLICA_DB_USER:-} + REPLICA_DB_PASS: ${REPLICA_DB_PASS:-} S3_ENDPOINT: ${S3_ENDPOINT:?S3_ENDPOINT must be set} S3_BUCKET: ${S3_BUCKET:-jorgecuadros-documents} MINIO_ROOT_USER: ${MINIO_ROOT_USER:?MINIO_ROOT_USER must be set}