wip: ops admin panel + migration sync + crud/rbac phase-5 snapshot

Working-tree checkpoint of in-progress work carried across prior
sessions on the feat/crud-rbac branch, committed so it lands on the
remote alongside the CI changes.

- Operaciones admin panel: apps/api/src/ops (ingest upload, backup /
  restore / re-import jobs) wired into app.module + RBAC abilities, and
  the apps/web/src/app/operaciones page. docker-compose gets INGEST_DIR
  / BACKUP_DIR volumes; .gitignore excludes migration/ingest + backups.
- migration/sync.py plus transform_*.py / run_all / config / dbenv /
  blob_extract adjustments for the additive sync path.
- crud/rbac phase-5 web bits: AppShell, api/labels/types libs, globals.
- schema.prisma + PLAN/RESUME doc updates.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

apps/api/src/app.module.ts

@@ -9,6 +9,7 @@ import { PoliciesModule } from "./policies/policies.module";
 import { PropertiesModule } from "./properties/properties.module";
 import { BillingModule } from "./billing/billing.module";
 import { BankModule } from "./bank/bank.module";
+import { OpsModule } from "./ops/ops.module";
 import { AppController } from "./app.controller";
 
 @Module({
@@ -23,6 +24,7 @@ import { AppController } from "./app.controller";
     PropertiesModule,
     BillingModule,
     BankModule,
+    OpsModule,
   ],
   controllers: [AppController],
 })

apps/api/src/auth/abilities.ts

@@ -32,7 +32,8 @@ export type Ability =
   | "bank:create"
   | "bank:void"
   | "lookup:manage"
-  | "user:manage";
+  | "user:manage"
+  | "db:manage";
 
 /** Minimum role required for each ability. */
 export const ABILITY_MIN: Record<Ability, Role> = {
@@ -51,6 +52,7 @@ export const ABILITY_MIN: Record<Ability, Role> = {
   "bank:void": "MANAGER",
   "lookup:manage": "MANAGER",
   "user:manage": "ADMIN",
+  "db:manage": "ADMIN",
 };
 
 export const ALL_ABILITIES = Object.keys(ABILITY_MIN) as Ability[];

apps/api/src/ops/ops.controller.ts

@@ -0,0 +1,120 @@
+import {
+  Body,
+  Controller,
+  Delete,
+  Get,
+  Param,
+  Post,
+  Req,
+  Res,
+  StreamableFile,
+  UploadedFile,
+  UseGuards,
+  UseInterceptors,
+} from "@nestjs/common";
+import { FileInterceptor } from "@nestjs/platform-express";
+import { Request, Response } from "express";
+import { AuthenticatedGuard } from "../auth/authenticated.guard";
+import { AbilityGuard } from "../auth/ability.guard";
+import { RequireAbility } from "../auth/require-ability.decorator";
+import { AuditService } from "../common/audit.service";
+import { OpsService } from "./ops.service";
+import { StartJobDto } from "./start-job.dto";
+
+/** Every route is ADMIN-only (ability "db:manage"). */
+@UseGuards(AuthenticatedGuard, AbilityGuard)
+@RequireAbility("db:manage")
+@Controller("ops")
+export class OpsController {
+  constructor(
+    private readonly ops: OpsService,
+    private readonly audit: AuditService,
+  ) {}
+
+  private actingId(req: Request): string {
+    return (req.user as { id: string }).id;
+  }
+
+  /* ------------------------------------------------------------- ingest */
+
+  @Get("ingest")
+  listIngest() {
+    return this.ops.listIngest();
+  }
+
+  @Post("ingest/:name")
+  @UseInterceptors(
+    FileInterceptor("file", { limits: { fileSize: 500 * 1024 * 1024 } }),
+  )
+  async uploadIngest(
+    @Param("name") name: string,
+    @UploadedFile() file: { buffer: Buffer; size: number } | undefined,
+    @Req() req: Request,
+  ) {
+    if (!file) throw new Error("No se recibió ningún archivo.");
+    await this.ops.saveIngest(name, file.buffer);
+    void this.audit.log(this.actingId(req), "ops.ingest.upload", {
+      name,
+      size: file.size,
+    });
+    return { ok: true };
+  }
+
+  @Delete("ingest/:name")
+  async deleteIngest(@Param("name") name: string, @Req() req: Request) {
+    await this.ops.deleteIngest(name);
+    void this.audit.log(this.actingId(req), "ops.ingest.delete", { name });
+    return { ok: true };
+  }
+
+  /* ------------------------------------------------------------ backups */
+
+  @Get("backups")
+  listBackups() {
+    return this.ops.listBackups();
+  }
+
+  @Get("backups/:name/download")
+  download(
+    @Param("name") name: string,
+    @Res({ passthrough: true }) res: Response,
+  ): StreamableFile {
+    const { stream, name: safe } = this.ops.backupStream(name);
+    res.set({
+      "Content-Type": "application/gzip",
+      "Content-Disposition": `attachment; filename="${safe}"`,
+    });
+    return new StreamableFile(stream);
+  }
+
+  @Delete("backups/:name")
+  async deleteBackup(@Param("name") name: string, @Req() req: Request) {
+    await this.ops.deleteBackup(name);
+    void this.audit.log(this.actingId(req), "ops.backup.delete", { name });
+    return { ok: true };
+  }
+
+  /* --------------------------------------------------------------- jobs */
+
+  @Get("jobs")
+  listJobs() {
+    return this.ops.listJobs();
+  }
+
+  @Get("jobs/:id")
+  getJob(@Param("id") id: string) {
+    return this.ops.getJob(id);
+  }
+
+  @Post("jobs")
+  async startJob(@Body() dto: StartJobDto, @Req() req: Request) {
+    const userId = this.actingId(req);
+    const job = await this.ops.startJob(dto.kind, { file: dto.file }, userId);
+    void this.audit.log(userId, "ops.job.start", {
+      jobId: job.id,
+      kind: dto.kind,
+      file: dto.file,
+    });
+    return job;
+  }
+}

apps/api/src/ops/ops.module.ts

@@ -0,0 +1,9 @@
+import { Module } from "@nestjs/common";
+import { OpsController } from "./ops.controller";
+import { OpsService } from "./ops.service";
+
+@Module({
+  controllers: [OpsController],
+  providers: [OpsService],
+})
+export class OpsModule {}

apps/api/src/ops/ops.service.ts

@@ -0,0 +1,361 @@
+import {
+  BadRequestException,
+  ConflictException,
+  Injectable,
+  Logger,
+  NotFoundException,
+  OnModuleInit,
+} from "@nestjs/common";
+import { spawn } from "node:child_process";
+import { createReadStream, promises as fs } from "node:fs";
+import * as path from "node:path";
+import { OpsJobKind } from "@jorgecuadros/database";
+import { PrismaService } from "../prisma/prisma.service";
+
+/**
+ * Admin database operations. Everything long-running (mysqldump, mysql restore,
+ * the Python migration) runs as a detached child process recorded as one OpsJob
+ * row whose `log` is appended as the process talks; the web polls that row.
+ *
+ * Only ONE mutating job runs at a time (a RUNNING row blocks a new start) — a
+ * restore or re-import racing a migration would corrupt the database.
+ */
+
+/** The four legacy Access files. Uploads are allowlisted to exactly these
+ *  names so an ingest write can never land at an arbitrary path. */
+export const INGEST_FILES = [
+  "UTILITIES.accdb",
+  "SEGUROS 16.mdb",
+  "SEGUROS 16_be.mdb",
+  "SCOTHIA.mdb",
+] as const;
+export type IngestName = (typeof INGEST_FILES)[number];
+
+interface MysqlConn {
+  host: string;
+  port: string;
+  user: string;
+  password: string;
+  database: string;
+}
+
+@Injectable()
+export class OpsService implements OnModuleInit {
+  private readonly logger = new Logger(OpsService.name);
+
+  private readonly migrationDir =
+    process.env.MIGRATION_DIR ?? path.resolve(process.cwd(), "migration");
+  private readonly ingestDir =
+    process.env.INGEST_DIR ?? path.join(this.migrationDir, "ingest");
+  private readonly backupDir =
+    process.env.BACKUP_DIR ?? path.join(this.migrationDir, "backups");
+  private readonly migrationEnv = process.env.MIGRATION_ENV ?? "dev";
+
+  constructor(private readonly prisma: PrismaService) {}
+
+  async onModuleInit(): Promise<void> {
+    await fs.mkdir(this.ingestDir, { recursive: true });
+    await fs.mkdir(this.backupDir, { recursive: true });
+  }
+
+  /* -------------------------------------------------------------- ingest */
+
+  private assertIngestName(name: string): IngestName {
+    if (!INGEST_FILES.includes(name as IngestName)) {
+      throw new BadRequestException(
+        `Archivo no permitido. Debe ser uno de: ${INGEST_FILES.join(", ")}`,
+      );
+    }
+    return name as IngestName;
+  }
+
+  async listIngest(): Promise<
+    { name: string; present: boolean; size: number | null; modifiedAt: string | null }[]
+  > {
+    return Promise.all(
+      INGEST_FILES.map(async (name) => {
+        try {
+          const st = await fs.stat(path.join(this.ingestDir, name));
+          return {
+            name,
+            present: true,
+            size: st.size,
+            modifiedAt: st.mtime.toISOString(),
+          };
+        } catch {
+          return { name, present: false, size: null, modifiedAt: null };
+        }
+      }),
+    );
+  }
+
+  async saveIngest(name: string, data: Buffer): Promise<void> {
+    const safe = this.assertIngestName(name);
+    await fs.writeFile(path.join(this.ingestDir, safe), data);
+  }
+
+  async deleteIngest(name: string): Promise<void> {
+    const safe = this.assertIngestName(name);
+    await fs.rm(path.join(this.ingestDir, safe), { force: true });
+  }
+
+  /* ------------------------------------------------------------- backups */
+
+  private assertBackupName(name: string): string {
+    // No path separators, must be a produced backup file.
+    if (!/^[A-Za-z0-9._-]+\.sql\.gz$/.test(name)) {
+      throw new BadRequestException("Nombre de respaldo inválido.");
+    }
+    return name;
+  }
+
+  async listBackups(): Promise<
+    { name: string; size: number; createdAt: string }[]
+  > {
+    let names: string[];
+    try {
+      names = await fs.readdir(this.backupDir);
+    } catch {
+      return [];
+    }
+    const rows = await Promise.all(
+      names
+        .filter((n) => n.endsWith(".sql.gz"))
+        .map(async (name) => {
+          const st = await fs.stat(path.join(this.backupDir, name));
+          return { name, size: st.size, createdAt: st.mtime.toISOString() };
+        }),
+    );
+    return rows.sort((a, b) => b.createdAt.localeCompare(a.createdAt));
+  }
+
+  backupStream(name: string) {
+    const safe = this.assertBackupName(name);
+    const full = path.join(this.backupDir, safe);
+    return { stream: createReadStream(full), name: safe };
+  }
+
+  async deleteBackup(name: string): Promise<void> {
+    const safe = this.assertBackupName(name);
+    await fs.rm(path.join(this.backupDir, safe), { force: true });
+  }
+
+  /* ---------------------------------------------------------------- jobs */
+
+  listJobs(limit = 20) {
+    return this.prisma.opsJob.findMany({
+      orderBy: { startedAt: "desc" },
+      take: limit,
+    });
+  }
+
+  async getJob(id: string) {
+    const job = await this.prisma.opsJob.findUnique({ where: { id } });
+    if (!job) throw new NotFoundException("Trabajo no encontrado.");
+    return job;
+  }
+
+  /**
+   * Start a mutating op. Refuses if another job is already RUNNING. Returns the
+   * new job row immediately; the process runs on in the background and appends
+   * to `log` until it exits.
+   */
+  async startJob(
+    kind: OpsJobKind,
+    params: Record<string, unknown>,
+    userId: string | undefined,
+  ) {
+    const running = await this.prisma.opsJob.count({ where: { status: "RUNNING" } });
+    if (running > 0) {
+      throw new ConflictException(
+        "Ya hay una operación en curso. Espere a que termine.",
+      );
+    }
+
+    const conn = this.parseDbUrl();
+    const { cmd, resolvedParams } = await this.buildCommand(kind, params, conn);
+
+    const job = await this.prisma.opsJob.create({
+      data: {
+        kind,
+        status: "RUNNING",
+        log: "",
+        params: resolvedParams as object,
+        createdById: userId,
+      },
+    });
+
+    this.run(job.id, cmd, conn.password);
+    return job;
+  }
+
+  /* --------------------------------------------------------- internals */
+
+  private parseDbUrl(): MysqlConn {
+    const raw = process.env.DATABASE_URL;
+    if (!raw) throw new BadRequestException("DATABASE_URL no está configurada.");
+    const u = new URL(raw);
+    return {
+      host: u.hostname,
+      port: u.port || "3306",
+      user: decodeURIComponent(u.username),
+      password: decodeURIComponent(u.password),
+      database: u.pathname.replace(/^\//, ""),
+    };
+  }
+
+  /** mysql/mysqldump connection flags. The password goes through MYSQL_PWD in
+   *  the child env, never on the command line (which would leak via `ps`). */
+  private connFlags(c: MysqlConn): string {
+    return `--host=${c.host} --port=${c.port} --user=${shq(c.user)}`;
+  }
+
+  private timestamp(): string {
+    return new Date().toISOString().replace(/[:.]/g, "-").replace("T", "_").slice(0, 19);
+  }
+
+  private async buildCommand(
+    kind: OpsJobKind,
+    params: Record<string, unknown>,
+    conn: MysqlConn,
+  ): Promise<{ cmd: string; resolvedParams: Record<string, unknown> }> {
+    const flags = this.connFlags(conn);
+    const db = shq(conn.database);
+
+    if (kind === "BACKUP") {
+      const file = `backup-${this.migrationEnv}-${this.timestamp()}.sql.gz`;
+      const out = shq(path.join(this.backupDir, file));
+      return {
+        cmd: `mysqldump ${flags} --single-transaction --routines --triggers --no-tablespaces ${db} | gzip -c > ${out}`,
+        resolvedParams: { file },
+      };
+    }
+
+    if (kind === "RESTORE") {
+      const name = this.assertBackupName(String(params.file ?? ""));
+      const full = path.join(this.backupDir, name);
+      await fs.access(full).catch(() => {
+        throw new NotFoundException(`Respaldo no encontrado: ${name}`);
+      });
+      return {
+        cmd: `gunzip -c ${shq(full)} | mysql ${flags} ${db}`,
+        resolvedParams: { file: name },
+      };
+    }
+
+    if (kind === "SYNC") {
+      const file = `pre-sync-${this.migrationEnv}-${this.timestamp()}.sql.gz`;
+      const out = shq(path.join(this.backupDir, file));
+      const py = await this.pythonBin();
+      const runAll = shq(path.join(this.migrationDir, "run_all.py"));
+      const cmd =
+        `echo '== Respaldo de seguridad previo ==' && ` +
+        `mysqldump ${flags} --single-transaction --routines --triggers --no-tablespaces ${db} | gzip -c > ${out} && ` +
+        `echo '== Sincronización aditiva desde carpeta de ingesta ==' && ` +
+        `${shq(py)} ${runAll} --env ${shq(this.migrationEnv)} --sync`;
+      return { cmd, resolvedParams: { safetyBackup: file } };
+    }
+
+    if (kind === "REIMPORT") {
+      // Safety backup first, then a full truncate+rebuild from the ingest files.
+      const file = `pre-reimport-${this.migrationEnv}-${this.timestamp()}.sql.gz`;
+      const out = shq(path.join(this.backupDir, file));
+      const py = await this.pythonBin();
+      const runAll = shq(path.join(this.migrationDir, "run_all.py"));
+      const cmd =
+        `echo '== Respaldo de seguridad previo ==' && ` +
+        `mysqldump ${flags} --single-transaction --routines --triggers --no-tablespaces ${db} | gzip -c > ${out} && ` +
+        `echo '== Reimportación desde carpeta de ingesta ==' && ` +
+        `${shq(py)} ${runAll} --env ${shq(this.migrationEnv)} --stage`;
+      return { cmd, resolvedParams: { safetyBackup: file } };
+    }
+
+    throw new BadRequestException(`Operación no soportada: ${kind}`);
+  }
+
+  /** Prefer the migration venv python if it exists (local dev), else system. */
+  private async pythonBin(): Promise<string> {
+    const venv = path.join(this.migrationDir, ".venv", "bin", "python");
+    try {
+      await fs.access(venv);
+      return venv;
+    } catch {
+      return process.env.PYTHON_BIN ?? "python3";
+    }
+  }
+
+  private run(jobId: string, cmd: string, password: string): void {
+    const child = spawn("sh", ["-c", cmd], {
+      cwd: this.migrationDir,
+      env: {
+        ...process.env,
+        MYSQL_PWD: password,
+        INGEST_DIR: this.ingestDir,
+        BACKUP_DIR: this.backupDir,
+      },
+    });
+
+    let buffer = "";
+    let pending = "";
+    let flushing = false;
+    let flushTimer: NodeJS.Timeout | null = null;
+
+    const flush = async () => {
+      if (flushing || !pending) return;
+      flushing = true;
+      const chunk = pending;
+      pending = "";
+      try {
+        await this.prisma.opsJob.update({
+          where: { id: jobId },
+          data: { log: { set: buffer } },
+        });
+      } catch (e) {
+        this.logger.warn(`ops job ${jobId} log flush failed: ${String(e)}`);
+      } finally {
+        flushing = false;
+        void chunk;
+      }
+    };
+
+    const onData = (d: Buffer) => {
+      const text = d.toString();
+      buffer += text;
+      pending += text;
+      if (!flushTimer) {
+        flushTimer = setTimeout(() => {
+          flushTimer = null;
+          void flush();
+        }, 1000);
+      }
+    };
+
+    child.stdout.on("data", onData);
+    child.stderr.on("data", onData);
+
+    const finalize = async (status: "SUCCESS" | "FAILED", tail: string) => {
+      if (flushTimer) clearTimeout(flushTimer);
+      buffer += tail;
+      await this.prisma.opsJob
+        .update({
+          where: { id: jobId },
+          data: { status, log: { set: buffer }, finishedAt: new Date() },
+        })
+        .catch((e) => this.logger.error(`ops job ${jobId} finalize failed: ${String(e)}`));
+    };
+
+    child.on("error", (err) => {
+      void finalize("FAILED", `\n[proceso no pudo iniciar] ${err.message}\n`);
+    });
+
+    child.on("close", (code) => {
+      if (code === 0) void finalize("SUCCESS", `\n[completado con éxito]\n`);
+      else void finalize("FAILED", `\n[terminó con código ${code}]\n`);
+    });
+  }
+}
+
+/** Single-quote a value for a POSIX shell command. */
+function shq(v: string): string {
+  return `'${v.replace(/'/g, `'\\''`)}'`;
+}

apps/api/src/ops/start-job.dto.ts

@@ -0,0 +1,12 @@
+import { IsEnum, IsOptional, IsString } from "class-validator";
+import { OpsJobKind } from "@jorgecuadros/database";
+
+export class StartJobDto {
+  @IsEnum(OpsJobKind)
+  kind!: OpsJobKind;
+
+  /** Target backup filename — required for RESTORE, ignored otherwise. */
+  @IsOptional()
+  @IsString()
+  file?: string;
+}