diff --git a/apps/api/src/ops/job-progress.spec.ts b/apps/api/src/ops/job-progress.spec.ts new file mode 100644 index 0000000..be6e30f --- /dev/null +++ b/apps/api/src/ops/job-progress.spec.ts @@ -0,0 +1,76 @@ +import { jobProgress } from "./ops.service"; + +/** Shape run_all.py emits, with the shell trace lines it interleaves. */ +const line = (i: number, n: number, name: string) => + `[paso ${i}/${n}] ${name}\n+ /repo/migration/.venv/bin/python /repo/migration/${name} --env prod\n[${name}] target env: prod\n validation: OK`; + +describe("jobProgress", () => { + it("returns null before any step marker appears", () => { + // The safety backup runs before run_all.py, so this is the real state for + // the first stretch of every REIMPORT. + expect(jobProgress("== Respaldo de seguridad previo ==\ntablas capturadas: 39", "RUNNING")).toBeNull(); + }); + + it("returns null for jobs that have no steps at all", () => { + // BACKUP/RESTORE are a single mysqldump; a fabricated percentage would be + // worse than none. + expect(jobProgress("mysqldump ... done", "SUCCESS")).toBeNull(); + }); + + it("tracks the most recent marker, not the first", () => { + const log = [line(1, 9, "transform_customers.py"), line(2, 9, "transform_properties.py")].join("\n"); + const p = jobProgress(log, "RUNNING"); + expect(p).toMatchObject({ step: 2, total: 9, name: "transform_properties.py" }); + }); + + /** + * The point of the whole feature. While RUNNING, step i is IN PROGRESS, so + * only i-1 are done. Counting i as complete would show 100% while the final + * and slowest step (blob_extract) is still working. + */ + it("does not claim a running step is finished", () => { + expect(jobProgress(line(1, 9, "transform_customers.py"), "RUNNING")?.percent).toBe(0); + expect(jobProgress(line(9, 9, "blob_extract.py"), "RUNNING")?.percent).toBe(88); + }); + + it("reaches 100 only once the job is no longer running", () => { + expect(jobProgress(line(9, 9, "blob_extract.py"), "SUCCESS")?.percent).toBe(100); + }); + + /** A job that died mid-way must report where it died, not 100%. */ + it("reports the failed step rather than completion", () => { + const p = jobProgress(line(5, 9, "transform_transactions.py"), "FAILED"); + expect(p).toMatchObject({ step: 5, total: 9 }); + expect(p!.percent).toBe(55); + }); + + it("handles the 8-step SYNC list as well as the 9-step REIMPORT one", () => { + expect(jobProgress(line(8, 8, "transform_bank.py"), "SUCCESS")?.percent).toBe(100); + expect(jobProgress(line(4, 8, "transform_policies.py"), "RUNNING")?.percent).toBe(37); + }); + + /** + * Captured verbatim from `run_all.run(..., step=8, total=9)`. This is the + * contract between the Python and this parser; if run_all.py's format + * changes, this fails rather than the panel silently showing no progress. + */ + it("parses the exact line run_all.py emits", () => { + const real = + "[paso 8/9] transform_bank.py\n+ /repo/migration/.venv/bin/python /repo/migration/transform_bank.py --env prod"; + expect(jobProgress(real, "RUNNING")).toMatchObject({ + step: 8, + total: 9, + name: "transform_bank.py", + percent: 77, + }); + }); + + it("ignores a malformed marker instead of reporting NaN", () => { + expect(jobProgress("[paso 3/0] x.py", "RUNNING")).toBeNull(); + }); + + /** The marker must be at line start so log text quoting it cannot spoof it. */ + it("does not match a marker embedded mid-line", () => { + expect(jobProgress("some output mentioning [paso 4/9] fake.py", "RUNNING")).toBeNull(); + }); +}); diff --git a/apps/api/src/ops/ops.service.ts b/apps/api/src/ops/ops.service.ts index d626922..3d754cf 100644 --- a/apps/api/src/ops/ops.service.ts +++ b/apps/api/src/ops/ops.service.ts @@ -212,7 +212,9 @@ export class OpsService implements OnModuleInit { async getJob(id: string) { const job = await this.prisma.opsJob.findUnique({ where: { id } }); if (!job) throw new NotFoundException("Trabajo no encontrado."); - return job; + // Derived, never stored: the log is the single source of truth for how far + // a job got, so progress cannot drift out of sync with it. + return { ...job, progress: jobProgress(job.log, job.status) }; } /** @@ -514,3 +516,52 @@ export class OpsService implements OnModuleInit { function shq(v: string): string { return `'${v.replace(/'/g, `'\\''`)}'`; } + +/** Progress derived from a job's log. Null when the job reports no steps. */ +export interface JobProgress { + /** 1-based index of the step currently running (or last reached). */ + step: number; + total: number; + /** Script name, e.g. "transform_bank.py". */ + name: string; + /** 0..100, floored. 100 only once the job is no longer RUNNING. */ + percent: number; +} + +/** + * Parse the "[paso i/N] name" markers migration/run_all.py emits. + * + * Progress is DERIVED from the log rather than tracked in a column: the log is + * already the record of what happened, and a separate counter could disagree + * with it — which is exactly the confusion a progress display is supposed to + * remove. run_all.py owns the step count, so adding a step cannot desync this. + * + * BACKUP and RESTORE are a single mysqldump with no steps, so they return null + * and the UI shows an indeterminate spinner. Reporting a fabricated percentage + * for them would be worse than showing none. + */ +export function jobProgress( + log: string, + status: string, +): JobProgress | null { + // Last marker wins: the log grows, and the newest line is the current step. + const matches = [...log.matchAll(/^\[paso (\d+)\/(\d+)\] (\S+)/gm)]; + const last = matches[matches.length - 1]; + if (!last) return null; + + const step = Number(last[1]); + const total = Number(last[2]); + if (!Number.isFinite(step) || !Number.isFinite(total) || total <= 0) return null; + + // While RUNNING, step i means i is IN PROGRESS, not finished — so report + // (i-1) completed. Claiming 100% while the last step is still working is the + // classic progress-bar lie, and here the last step (blob_extract) is also the + // slowest, so it would sit at "100%" for the longest stretch of the job. + const done = status === "RUNNING" ? step - 1 : step; + return { + step, + total, + name: last[3], + percent: Math.max(0, Math.min(100, Math.floor((done / total) * 100))), + }; +} diff --git a/apps/web/src/app/operaciones/page.tsx b/apps/web/src/app/operaciones/page.tsx index 5489551..8736ef8 100644 --- a/apps/web/src/app/operaciones/page.tsx +++ b/apps/web/src/app/operaciones/page.tsx @@ -225,6 +225,7 @@ function Operaciones() { )} +
{activeJob.log || "Iniciando…"}
)} @@ -699,3 +700,70 @@ function KV({ label, value }: { label: string; value: string | null | undefined ); } + +/** + * Step progress for a running migration. + * + * Only REIMPORT and SYNC report steps; BACKUP and RESTORE are a single + * mysqldump, so they render nothing here rather than a made-up bar — the + * spinner in the heading already says "working". + * + * The safety backup runs before the migration, so `progress` is null for the + * first stretch of every REIMPORT. That phase is named explicitly instead of + * showing 0%, which would read as "stuck". + */ +function JobProgressBar({ job }: { job: OpsJob }) { + const running = job.status === "RUNNING"; + const p = job.progress; + + if (!p) { + if (!running) return null; + return ( +

+ Respaldo de seguridad previo… +

+ ); + } + + return ( +
+
+ + Paso {p.step} de {p.total} — {p.name} + + + {p.percent}% + +
+
+
+
+
+ ); +} diff --git a/apps/web/src/lib/types.ts b/apps/web/src/lib/types.ts index 57f737f..c613514 100644 --- a/apps/web/src/lib/types.ts +++ b/apps/web/src/lib/types.ts @@ -61,6 +61,14 @@ export interface UserRow { export type OpsJobKind = "BACKUP" | "RESTORE" | "REIMPORT" | "SYNC"; export type OpsJobStatus = "RUNNING" | "SUCCESS" | "FAILED"; +/** Derived from the job log by the API; null for jobs with no step markers. */ +export interface JobProgress { + step: number; + total: number; + name: string; + percent: number; +} + export interface OpsJob { id: string; kind: OpsJobKind; @@ -70,6 +78,8 @@ export interface OpsJob { createdById: string | null; startedAt: string; finishedAt: string | null; + /** Only present on getOpsJob (the polled endpoint), not on the list. */ + progress?: JobProgress | null; } /** diff --git a/migration/run_all.py b/migration/run_all.py index d685626..1e68f34 100644 --- a/migration/run_all.py +++ b/migration/run_all.py @@ -78,7 +78,13 @@ SYNC_STEPS = [ ] -def run(cmd: list[str]) -> None: +def run(cmd: list[str], step: int | None = None, total: int | None = None) -> None: + # The "[paso i/N] name" marker is a contract with the Operaciones screen, + # which parses the last one to show progress. Emitting it here rather than + # letting the UI count STEPS itself keeps the two from drifting when a step + # is added — the number of steps is only ever stated in this file. + if step is not None and total is not None: + print(f"[paso {step}/{total}] {Path(cmd[1]).name}", flush=True) print("+ " + " ".join(cmd), flush=True) r = subprocess.run(cmd) if r.returncode: @@ -97,11 +103,12 @@ def main() -> None: if args.stage: run([PY, str(HERE / "load_staging.py"), "--output-dir", str(HERE / "output")]) - for step in SYNC_STEPS if args.sync else STEPS: + steps = SYNC_STEPS if args.sync else STEPS + for i, step in enumerate(steps, start=1): cmd = [PY, str(HERE / step), "--env", args.env] if args.sync: cmd.append("--sync") - run(cmd) + run(cmd, step=i, total=len(steps)) print(f"\n✓ migration complete for env={args.env}")