wip: ops admin panel + migration sync + crud/rbac phase-5 snapshot

Working-tree checkpoint of in-progress work carried across prior sessions on the feat/crud-rbac branch, committed so it lands on the remote alongside the CI changes. - Operaciones admin panel: apps/api/src/ops (ingest upload, backup / restore / re-import jobs) wired into app.module + RBAC abilities, and the apps/web/src/app/operaciones page. docker-compose gets INGEST_DIR / BACKUP_DIR volumes; .gitignore excludes migration/ingest + backups. - migration/sync.py plus transform_*.py / run_all / config / dbenv / blob_extract adjustments for the additive sync path. - crud/rbac phase-5 web bits: AppShell, api/labels/types libs, globals. - schema.prisma + PLAN/RESUME doc updates. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-23 19:01:36 -07:00
parent 6ad0993a71
commit f1ef1c70b3
27 changed files with 1480 additions and 107 deletions
@@ -0,0 +1,361 @@
+import {
+  BadRequestException,
+  ConflictException,
+  Injectable,
+  Logger,
+  NotFoundException,
+  OnModuleInit,
+} from "@nestjs/common";
+import { spawn } from "node:child_process";
+import { createReadStream, promises as fs } from "node:fs";
+import * as path from "node:path";
+import { OpsJobKind } from "@jorgecuadros/database";
+import { PrismaService } from "../prisma/prisma.service";
+
+/**
+ * Admin database operations. Everything long-running (mysqldump, mysql restore,
+ * the Python migration) runs as a detached child process recorded as one OpsJob
+ * row whose `log` is appended as the process talks; the web polls that row.
+ *
+ * Only ONE mutating job runs at a time (a RUNNING row blocks a new start) — a
+ * restore or re-import racing a migration would corrupt the database.
+ */
+
+/** The four legacy Access files. Uploads are allowlisted to exactly these
+ *  names so an ingest write can never land at an arbitrary path. */
+export const INGEST_FILES = [
+  "UTILITIES.accdb",
+  "SEGUROS 16.mdb",
+  "SEGUROS 16_be.mdb",
+  "SCOTHIA.mdb",
+] as const;
+export type IngestName = (typeof INGEST_FILES)[number];
+
+interface MysqlConn {
+  host: string;
+  port: string;
+  user: string;
+  password: string;
+  database: string;
+}
+
+@Injectable()
+export class OpsService implements OnModuleInit {
+  private readonly logger = new Logger(OpsService.name);
+
+  private readonly migrationDir =
+    process.env.MIGRATION_DIR ?? path.resolve(process.cwd(), "migration");
+  private readonly ingestDir =
+    process.env.INGEST_DIR ?? path.join(this.migrationDir, "ingest");
+  private readonly backupDir =
+    process.env.BACKUP_DIR ?? path.join(this.migrationDir, "backups");
+  private readonly migrationEnv = process.env.MIGRATION_ENV ?? "dev";
+
+  constructor(private readonly prisma: PrismaService) {}
+
+  async onModuleInit(): Promise<void> {
+    await fs.mkdir(this.ingestDir, { recursive: true });
+    await fs.mkdir(this.backupDir, { recursive: true });
+  }
+
+  /* -------------------------------------------------------------- ingest */
+
+  private assertIngestName(name: string): IngestName {
+    if (!INGEST_FILES.includes(name as IngestName)) {
+      throw new BadRequestException(
+        `Archivo no permitido. Debe ser uno de: ${INGEST_FILES.join(", ")}`,
+      );
+    }
+    return name as IngestName;
+  }
+
+  async listIngest(): Promise<
+    { name: string; present: boolean; size: number | null; modifiedAt: string | null }[]
+  > {
+    return Promise.all(
+      INGEST_FILES.map(async (name) => {
+        try {
+          const st = await fs.stat(path.join(this.ingestDir, name));
+          return {
+            name,
+            present: true,
+            size: st.size,
+            modifiedAt: st.mtime.toISOString(),
+          };
+        } catch {
+          return { name, present: false, size: null, modifiedAt: null };
+        }
+      }),
+    );
+  }
+
+  async saveIngest(name: string, data: Buffer): Promise<void> {
+    const safe = this.assertIngestName(name);
+    await fs.writeFile(path.join(this.ingestDir, safe), data);
+  }
+
+  async deleteIngest(name: string): Promise<void> {
+    const safe = this.assertIngestName(name);
+    await fs.rm(path.join(this.ingestDir, safe), { force: true });
+  }
+
+  /* ------------------------------------------------------------- backups */
+
+  private assertBackupName(name: string): string {
+    // No path separators, must be a produced backup file.
+    if (!/^[A-Za-z0-9._-]+\.sql\.gz$/.test(name)) {
+      throw new BadRequestException("Nombre de respaldo inválido.");
+    }
+    return name;
+  }
+
+  async listBackups(): Promise<
+    { name: string; size: number; createdAt: string }[]
+  > {
+    let names: string[];
+    try {
+      names = await fs.readdir(this.backupDir);
+    } catch {
+      return [];
+    }
+    const rows = await Promise.all(
+      names
+        .filter((n) => n.endsWith(".sql.gz"))
+        .map(async (name) => {
+          const st = await fs.stat(path.join(this.backupDir, name));
+          return { name, size: st.size, createdAt: st.mtime.toISOString() };
+        }),
+    );
+    return rows.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
+  }
+
+  backupStream(name: string) {
+    const safe = this.assertBackupName(name);
+    const full = path.join(this.backupDir, safe);
+    return { stream: createReadStream(full), name: safe };
+  }
+
+  async deleteBackup(name: string): Promise<void> {
+    const safe = this.assertBackupName(name);
+    await fs.rm(path.join(this.backupDir, safe), { force: true });
+  }
+
+  /* ---------------------------------------------------------------- jobs */
+
+  listJobs(limit = 20) {
+    return this.prisma.opsJob.findMany({
+      orderBy: { startedAt: "desc" },
+      take: limit,
+    });
+  }
+
+  async getJob(id: string) {
+    const job = await this.prisma.opsJob.findUnique({ where: { id } });
+    if (!job) throw new NotFoundException("Trabajo no encontrado.");
+    return job;
+  }
+
+  /**
+   * Start a mutating op. Refuses if another job is already RUNNING. Returns the
+   * new job row immediately; the process runs on in the background and appends
+   * to `log` until it exits.
+   */
+  async startJob(
+    kind: OpsJobKind,
+    params: Record<string, unknown>,
+    userId: string | undefined,
+  ) {
+    const running = await this.prisma.opsJob.count({ where: { status: "RUNNING" } });
+    if (running > 0) {
+      throw new ConflictException(
+        "Ya hay una operación en curso. Espere a que termine.",
+      );
+    }
+
+    const conn = this.parseDbUrl();
+    const { cmd, resolvedParams } = await this.buildCommand(kind, params, conn);
+
+    const job = await this.prisma.opsJob.create({
+      data: {
+        kind,
+        status: "RUNNING",
+        log: "",
+        params: resolvedParams as object,
+        createdById: userId,
+      },
+    });
+
+    this.run(job.id, cmd, conn.password);
+    return job;
+  }
+
+  /* --------------------------------------------------------- internals */
+
+  private parseDbUrl(): MysqlConn {
+    const raw = process.env.DATABASE_URL;
+    if (!raw) throw new BadRequestException("DATABASE_URL no está configurada.");
+    const u = new URL(raw);
+    return {
+      host: u.hostname,
+      port: u.port || "3306",
+      user: decodeURIComponent(u.username),
+      password: decodeURIComponent(u.password),
+      database: u.pathname.replace(/^\//, ""),
+    };
+  }
+
+  /** mysql/mysqldump connection flags. The password goes through MYSQL_PWD in
+   *  the child env, never on the command line (which would leak via `ps`). */
+  private connFlags(c: MysqlConn): string {
+    return `--host=${c.host} --port=${c.port} --user=${shq(c.user)}`;
+  }
+
+  private timestamp(): string {
+    return new Date().toISOString().replace(/[:.]/g, "-").replace("T", "_").slice(0, 19);
+  }
+
+  private async buildCommand(
+    kind: OpsJobKind,
+    params: Record<string, unknown>,
+    conn: MysqlConn,
+  ): Promise<{ cmd: string; resolvedParams: Record<string, unknown> }> {
+    const flags = this.connFlags(conn);
+    const db = shq(conn.database);
+
+    if (kind === "BACKUP") {
+      const file = `backup-${this.migrationEnv}-${this.timestamp()}.sql.gz`;
+      const out = shq(path.join(this.backupDir, file));
+      return {
+        cmd: `mysqldump ${flags} --single-transaction --routines --triggers --no-tablespaces ${db} | gzip -c > ${out}`,
+        resolvedParams: { file },
+      };
+    }
+
+    if (kind === "RESTORE") {
+      const name = this.assertBackupName(String(params.file ?? ""));
+      const full = path.join(this.backupDir, name);
+      await fs.access(full).catch(() => {
+        throw new NotFoundException(`Respaldo no encontrado: ${name}`);
+      });
+      return {
+        cmd: `gunzip -c ${shq(full)} | mysql ${flags} ${db}`,
+        resolvedParams: { file: name },
+      };
+    }
+
+    if (kind === "SYNC") {
+      const file = `pre-sync-${this.migrationEnv}-${this.timestamp()}.sql.gz`;
+      const out = shq(path.join(this.backupDir, file));
+      const py = await this.pythonBin();
+      const runAll = shq(path.join(this.migrationDir, "run_all.py"));
+      const cmd =
+        `echo '== Respaldo de seguridad previo ==' && ` +
+        `mysqldump ${flags} --single-transaction --routines --triggers --no-tablespaces ${db} | gzip -c > ${out} && ` +
+        `echo '== Sincronización aditiva desde carpeta de ingesta ==' && ` +
+        `${shq(py)} ${runAll} --env ${shq(this.migrationEnv)} --sync`;
+      return { cmd, resolvedParams: { safetyBackup: file } };
+    }
+
+    if (kind === "REIMPORT") {
+      // Safety backup first, then a full truncate+rebuild from the ingest files.
+      const file = `pre-reimport-${this.migrationEnv}-${this.timestamp()}.sql.gz`;
+      const out = shq(path.join(this.backupDir, file));
+      const py = await this.pythonBin();
+      const runAll = shq(path.join(this.migrationDir, "run_all.py"));
+      const cmd =
+        `echo '== Respaldo de seguridad previo ==' && ` +
+        `mysqldump ${flags} --single-transaction --routines --triggers --no-tablespaces ${db} | gzip -c > ${out} && ` +
+        `echo '== Reimportación desde carpeta de ingesta ==' && ` +
+        `${shq(py)} ${runAll} --env ${shq(this.migrationEnv)} --stage`;
+      return { cmd, resolvedParams: { safetyBackup: file } };
+    }
+
+    throw new BadRequestException(`Operación no soportada: ${kind}`);
+  }
+
+  /** Prefer the migration venv python if it exists (local dev), else system. */
+  private async pythonBin(): Promise<string> {
+    const venv = path.join(this.migrationDir, ".venv", "bin", "python");
+    try {
+      await fs.access(venv);
+      return venv;
+    } catch {
+      return process.env.PYTHON_BIN ?? "python3";
+    }
+  }
+
+  private run(jobId: string, cmd: string, password: string): void {
+    const child = spawn("sh", ["-c", cmd], {
+      cwd: this.migrationDir,
+      env: {
+        ...process.env,
+        MYSQL_PWD: password,
+        INGEST_DIR: this.ingestDir,
+        BACKUP_DIR: this.backupDir,
+      },
+    });
+
+    let buffer = "";
+    let pending = "";
+    let flushing = false;
+    let flushTimer: NodeJS.Timeout | null = null;
+
+    const flush = async () => {
+      if (flushing || !pending) return;
+      flushing = true;
+      const chunk = pending;
+      pending = "";
+      try {
+        await this.prisma.opsJob.update({
+          where: { id: jobId },
+          data: { log: { set: buffer } },
+        });
+      } catch (e) {
+        this.logger.warn(`ops job ${jobId} log flush failed: ${String(e)}`);
+      } finally {
+        flushing = false;
+        void chunk;
+      }
+    };
+
+    const onData = (d: Buffer) => {
+      const text = d.toString();
+      buffer += text;
+      pending += text;
+      if (!flushTimer) {
+        flushTimer = setTimeout(() => {
+          flushTimer = null;
+          void flush();
+        }, 1000);
+      }
+    };
+
+    child.stdout.on("data", onData);
+    child.stderr.on("data", onData);
+
+    const finalize = async (status: "SUCCESS" | "FAILED", tail: string) => {
+      if (flushTimer) clearTimeout(flushTimer);
+      buffer += tail;
+      await this.prisma.opsJob
+        .update({
+          where: { id: jobId },
+          data: { status, log: { set: buffer }, finishedAt: new Date() },
+        })
+        .catch((e) => this.logger.error(`ops job ${jobId} finalize failed: ${String(e)}`));
+    };
+
+    child.on("error", (err) => {
+      void finalize("FAILED", `\n[proceso no pudo iniciar] ${err.message}\n`);
+    });
+
+    child.on("close", (code) => {
+      if (code === 0) void finalize("SUCCESS", `\n[completado con éxito]\n`);
+      else void finalize("FAILED", `\n[terminó con código ${code}]\n`);
+    });
+  }
+}
+
+/** Single-quote a value for a POSIX shell command. */
+function shq(v: string): string {
+  return `'${v.replace(/'/g, `'\\''`)}'`;
+}