Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4a929f7e7c | ||
|
|
66d0d071b0 |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@jorgecuadros/api",
|
||||
"version": "1.0.10",
|
||||
"version": "1.0.11",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "nest build",
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { jobProgress } from "./ops.service";
|
||||
|
||||
/** Shape run_all.py emits, with the shell trace lines it interleaves. */
|
||||
const line = (i: number, n: number, name: string) =>
|
||||
`[paso ${i}/${n}] ${name}\n+ /repo/migration/.venv/bin/python /repo/migration/${name} --env prod\n[${name}] target env: prod\n validation: OK`;
|
||||
|
||||
describe("jobProgress", () => {
|
||||
it("returns null before any step marker appears", () => {
|
||||
// The safety backup runs before run_all.py, so this is the real state for
|
||||
// the first stretch of every REIMPORT.
|
||||
expect(jobProgress("== Respaldo de seguridad previo ==\ntablas capturadas: 39", "RUNNING")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null for jobs that have no steps at all", () => {
|
||||
// BACKUP/RESTORE are a single mysqldump; a fabricated percentage would be
|
||||
// worse than none.
|
||||
expect(jobProgress("mysqldump ... done", "SUCCESS")).toBeNull();
|
||||
});
|
||||
|
||||
it("tracks the most recent marker, not the first", () => {
|
||||
const log = [line(1, 9, "transform_customers.py"), line(2, 9, "transform_properties.py")].join("\n");
|
||||
const p = jobProgress(log, "RUNNING");
|
||||
expect(p).toMatchObject({ step: 2, total: 9, name: "transform_properties.py" });
|
||||
});
|
||||
|
||||
/**
|
||||
* The point of the whole feature. While RUNNING, step i is IN PROGRESS, so
|
||||
* only i-1 are done. Counting i as complete would show 100% while the final
|
||||
* and slowest step (blob_extract) is still working.
|
||||
*/
|
||||
it("does not claim a running step is finished", () => {
|
||||
expect(jobProgress(line(1, 9, "transform_customers.py"), "RUNNING")?.percent).toBe(0);
|
||||
expect(jobProgress(line(9, 9, "blob_extract.py"), "RUNNING")?.percent).toBe(88);
|
||||
});
|
||||
|
||||
it("reaches 100 only once the job is no longer running", () => {
|
||||
expect(jobProgress(line(9, 9, "blob_extract.py"), "SUCCESS")?.percent).toBe(100);
|
||||
});
|
||||
|
||||
/** A job that died mid-way must report where it died, not 100%. */
|
||||
it("reports the failed step rather than completion", () => {
|
||||
const p = jobProgress(line(5, 9, "transform_transactions.py"), "FAILED");
|
||||
expect(p).toMatchObject({ step: 5, total: 9 });
|
||||
expect(p!.percent).toBe(55);
|
||||
});
|
||||
|
||||
it("handles the 8-step SYNC list as well as the 9-step REIMPORT one", () => {
|
||||
expect(jobProgress(line(8, 8, "transform_bank.py"), "SUCCESS")?.percent).toBe(100);
|
||||
expect(jobProgress(line(4, 8, "transform_policies.py"), "RUNNING")?.percent).toBe(37);
|
||||
});
|
||||
|
||||
/**
|
||||
* Captured verbatim from `run_all.run(..., step=8, total=9)`. This is the
|
||||
* contract between the Python and this parser; if run_all.py's format
|
||||
* changes, this fails rather than the panel silently showing no progress.
|
||||
*/
|
||||
it("parses the exact line run_all.py emits", () => {
|
||||
const real =
|
||||
"[paso 8/9] transform_bank.py\n+ /repo/migration/.venv/bin/python /repo/migration/transform_bank.py --env prod";
|
||||
expect(jobProgress(real, "RUNNING")).toMatchObject({
|
||||
step: 8,
|
||||
total: 9,
|
||||
name: "transform_bank.py",
|
||||
percent: 77,
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores a malformed marker instead of reporting NaN", () => {
|
||||
expect(jobProgress("[paso 3/0] x.py", "RUNNING")).toBeNull();
|
||||
});
|
||||
|
||||
/** The marker must be at line start so log text quoting it cannot spoof it. */
|
||||
it("does not match a marker embedded mid-line", () => {
|
||||
expect(jobProgress("some output mentioning [paso 4/9] fake.py", "RUNNING")).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -212,7 +212,9 @@ export class OpsService implements OnModuleInit {
|
||||
async getJob(id: string) {
|
||||
const job = await this.prisma.opsJob.findUnique({ where: { id } });
|
||||
if (!job) throw new NotFoundException("Trabajo no encontrado.");
|
||||
return job;
|
||||
// Derived, never stored: the log is the single source of truth for how far
|
||||
// a job got, so progress cannot drift out of sync with it.
|
||||
return { ...job, progress: jobProgress(job.log, job.status) };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -514,3 +516,52 @@ export class OpsService implements OnModuleInit {
|
||||
function shq(v: string): string {
|
||||
return `'${v.replace(/'/g, `'\\''`)}'`;
|
||||
}
|
||||
|
||||
/** Progress derived from a job's log. Null when the job reports no steps. */
|
||||
export interface JobProgress {
|
||||
/** 1-based index of the step currently running (or last reached). */
|
||||
step: number;
|
||||
total: number;
|
||||
/** Script name, e.g. "transform_bank.py". */
|
||||
name: string;
|
||||
/** 0..100, floored. 100 only once the job is no longer RUNNING. */
|
||||
percent: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the "[paso i/N] name" markers migration/run_all.py emits.
|
||||
*
|
||||
* Progress is DERIVED from the log rather than tracked in a column: the log is
|
||||
* already the record of what happened, and a separate counter could disagree
|
||||
* with it — which is exactly the confusion a progress display is supposed to
|
||||
* remove. run_all.py owns the step count, so adding a step cannot desync this.
|
||||
*
|
||||
* BACKUP and RESTORE are a single mysqldump with no steps, so they return null
|
||||
* and the UI shows an indeterminate spinner. Reporting a fabricated percentage
|
||||
* for them would be worse than showing none.
|
||||
*/
|
||||
export function jobProgress(
|
||||
log: string,
|
||||
status: string,
|
||||
): JobProgress | null {
|
||||
// Last marker wins: the log grows, and the newest line is the current step.
|
||||
const matches = [...log.matchAll(/^\[paso (\d+)\/(\d+)\] (\S+)/gm)];
|
||||
const last = matches[matches.length - 1];
|
||||
if (!last) return null;
|
||||
|
||||
const step = Number(last[1]);
|
||||
const total = Number(last[2]);
|
||||
if (!Number.isFinite(step) || !Number.isFinite(total) || total <= 0) return null;
|
||||
|
||||
// While RUNNING, step i means i is IN PROGRESS, not finished — so report
|
||||
// (i-1) completed. Claiming 100% while the last step is still working is the
|
||||
// classic progress-bar lie, and here the last step (blob_extract) is also the
|
||||
// slowest, so it would sit at "100%" for the longest stretch of the job.
|
||||
const done = status === "RUNNING" ? step - 1 : step;
|
||||
return {
|
||||
step,
|
||||
total,
|
||||
name: last[3],
|
||||
percent: Math.max(0, Math.min(100, Math.floor((done / total) * 100))),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@jorgecuadros/web",
|
||||
"version": "1.0.10",
|
||||
"version": "1.0.11",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev -p 4500",
|
||||
|
||||
@@ -225,6 +225,7 @@ function Operaciones() {
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<JobProgressBar job={activeJob} />
|
||||
<pre className="ops-log">{activeJob.log || "Iniciando…"}</pre>
|
||||
</div>
|
||||
)}
|
||||
@@ -699,3 +700,70 @@ function KV({ label, value }: { label: string; value: string | null | undefined
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Step progress for a running migration.
|
||||
*
|
||||
* Only REIMPORT and SYNC report steps; BACKUP and RESTORE are a single
|
||||
* mysqldump, so they render nothing here rather than a made-up bar — the
|
||||
* spinner in the heading already says "working".
|
||||
*
|
||||
* The safety backup runs before the migration, so `progress` is null for the
|
||||
* first stretch of every REIMPORT. That phase is named explicitly instead of
|
||||
* showing 0%, which would read as "stuck".
|
||||
*/
|
||||
function JobProgressBar({ job }: { job: OpsJob }) {
|
||||
const running = job.status === "RUNNING";
|
||||
const p = job.progress;
|
||||
|
||||
if (!p) {
|
||||
if (!running) return null;
|
||||
return (
|
||||
<p className="inline-form-note" style={{ marginTop: 8 }}>
|
||||
Respaldo de seguridad previo…
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div style={{ marginTop: 10, marginBottom: 4 }}>
|
||||
<div
|
||||
className="row-actions"
|
||||
style={{ justifyContent: "space-between", marginBottom: 6 }}
|
||||
>
|
||||
<span className="inline-form-note" style={{ margin: 0 }}>
|
||||
Paso {p.step} de {p.total} — {p.name}
|
||||
</span>
|
||||
<span className="inline-form-note" style={{ margin: 0 }}>
|
||||
{p.percent}%
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
role="progressbar"
|
||||
aria-valuenow={p.percent}
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-label={`Paso ${p.step} de ${p.total}`}
|
||||
style={{
|
||||
height: 6,
|
||||
borderRadius: 999,
|
||||
background: "var(--line)",
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: `${p.percent}%`,
|
||||
height: "100%",
|
||||
borderRadius: 999,
|
||||
transition: "width 400ms ease",
|
||||
background:
|
||||
job.status === "FAILED"
|
||||
? "var(--negative)"
|
||||
: "var(--positive)",
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -61,6 +61,14 @@ export interface UserRow {
|
||||
export type OpsJobKind = "BACKUP" | "RESTORE" | "REIMPORT" | "SYNC";
|
||||
export type OpsJobStatus = "RUNNING" | "SUCCESS" | "FAILED";
|
||||
|
||||
/** Derived from the job log by the API; null for jobs with no step markers. */
|
||||
export interface JobProgress {
|
||||
step: number;
|
||||
total: number;
|
||||
name: string;
|
||||
percent: number;
|
||||
}
|
||||
|
||||
export interface OpsJob {
|
||||
id: string;
|
||||
kind: OpsJobKind;
|
||||
@@ -70,6 +78,8 @@ export interface OpsJob {
|
||||
createdById: string | null;
|
||||
startedAt: string;
|
||||
finishedAt: string | null;
|
||||
/** Only present on getOpsJob (the polled endpoint), not on the list. */
|
||||
progress?: JobProgress | null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+10
-3
@@ -78,7 +78,13 @@ SYNC_STEPS = [
|
||||
]
|
||||
|
||||
|
||||
def run(cmd: list[str]) -> None:
|
||||
def run(cmd: list[str], step: int | None = None, total: int | None = None) -> None:
|
||||
# The "[paso i/N] name" marker is a contract with the Operaciones screen,
|
||||
# which parses the last one to show progress. Emitting it here rather than
|
||||
# letting the UI count STEPS itself keeps the two from drifting when a step
|
||||
# is added — the number of steps is only ever stated in this file.
|
||||
if step is not None and total is not None:
|
||||
print(f"[paso {step}/{total}] {Path(cmd[1]).name}", flush=True)
|
||||
print("+ " + " ".join(cmd), flush=True)
|
||||
r = subprocess.run(cmd)
|
||||
if r.returncode:
|
||||
@@ -97,11 +103,12 @@ def main() -> None:
|
||||
if args.stage:
|
||||
run([PY, str(HERE / "load_staging.py"), "--output-dir", str(HERE / "output")])
|
||||
|
||||
for step in SYNC_STEPS if args.sync else STEPS:
|
||||
steps = SYNC_STEPS if args.sync else STEPS
|
||||
for i, step in enumerate(steps, start=1):
|
||||
cmd = [PY, str(HERE / step), "--env", args.env]
|
||||
if args.sync:
|
||||
cmd.append("--sync")
|
||||
run(cmd)
|
||||
run(cmd, step=i, total=len(steps))
|
||||
|
||||
print(f"\n✓ migration complete for env={args.env}")
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "jorgecuadros-platform",
|
||||
"version": "1.0.10",
|
||||
"version": "1.0.11",
|
||||
"private": true,
|
||||
"workspaces": [
|
||||
"apps/*",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@jorgecuadros/database",
|
||||
"version": "1.0.10",
|
||||
"version": "1.0.11",
|
||||
"private": true,
|
||||
"main": "generated/client/index.js",
|
||||
"types": "generated/client/index.d.ts",
|
||||
|
||||
Reference in New Issue
Block a user