From 7797c45e9f1b4453845a776db6d1884012d7b413 Mon Sep 17 00:00:00 2001
From: Ricardo Mancinas <rmancinas@freakma.net>
Date: Mon, 3 Aug 2026 17:22:15 -0700
Subject: [PATCH] fix(ops): fail orphaned RUNNING jobs at startup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Ops jobs run as a child of the API process, so no job can outlive it. When a
deploy landed 110 seconds into a REIMPORT, the child died and nothing was
left to finalize the row — it stayed RUNNING forever. Because startJob()
refuses to start while any RUNNING row exists, that one interrupted job
wedged the panel permanently with no way out from the UI; recovering it took
a manual UPDATE against the production database.

A fresh boot is proof that nothing survived, so this is unconditional rather
than filtered on age: "started recently" does not imply "still alive" here.

Rows are updated one at a time rather than with updateMany so the reason can
be APPENDED to the log. A job whose log simply stops mid-step with no
explanation is what made the first occurrence hard to diagnose.

Failure to reconcile is logged and swallowed: a wedged panel is bad, an API
that will not boot is worse.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 apps/api/src/ops/ops.service.ts | 49 +++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/apps/api/src/ops/ops.service.ts b/apps/api/src/ops/ops.service.ts
index 509379a..d626922 100644
--- a/apps/api/src/ops/ops.service.ts
+++ b/apps/api/src/ops/ops.service.ts
@@ -67,6 +67,55 @@ export class OpsService implements OnModuleInit {
   async onModuleInit(): Promise<void> {
     await fs.mkdir(this.ingestDir, { recursive: true });
     await fs.mkdir(this.backupDir, { recursive: true });
+    await this.reconcileOrphanedJobs();
+  }
+
+  /**
+   * Fail any job still marked RUNNING at startup.
+   *
+   * Jobs run as a child of THIS process, so no job can outlive it: if a row says
+   * RUNNING while we are booting, its process died with the previous instance
+   * and nothing will ever finalize it. Since startJob() refuses to start while
+   * any RUNNING row exists, one interrupted job wedges the panel permanently
+   * with no way out from the UI — it took a manual UPDATE against production to
+   * recover the first time this happened, when a deploy landed 110 seconds into
+   * a REIMPORT.
+   *
+   * Deliberately unconditional rather than filtered on age: "started recently"
+   * does not mean "still alive" here, and a fresh boot is proof enough that
+   * nothing survived.
+   */
+  private async reconcileOrphanedJobs(): Promise<void> {
+    try {
+      // Read then write one by one rather than updateMany: the log needs the
+      // reason APPENDED, and a job whose log just stops mid-step with no
+      // explanation is what made the first occurrence hard to diagnose.
+      const orphans = await this.prisma.opsJob.findMany({
+        where: { status: "RUNNING" },
+        select: { id: true, kind: true, log: true },
+      });
+      for (const job of orphans) {
+        await this.prisma.opsJob.update({
+          where: { id: job.id },
+          data: {
+            status: "FAILED",
+            finishedAt: new Date(),
+            log: {
+              set:
+                job.log +
+                "\n[interrumpido: el contenedor se reinició mientras el trabajo corría; " +
+                "el proceso hijo no sobrevive a un redespliegue. " +
+                "Vuelva a ejecutar la operación desde el principio.]\n",
+            },
+          },
+        });
+        this.logger.warn(`trabajo ${job.kind} ${job.id} quedó huérfano; marcado FAILED`);
+      }
+    } catch (e) {
+      // Never block startup on this. A failed reconcile leaves the panel
+      // wedged, which is bad, but an API that will not boot is worse.
+      this.logger.error(`no se pudieron reconciliar trabajos huérfanos: ${String(e)}`);
+    }
   }
 
   /* -------------------------------------------------------------- ingest */