feat(migration): refuse a full re-import that would delete native rows
A full run_all.py pass truncates and rebuilds every table it owns from the Access extract. That was harmless while the platform was a read-only mirror -- every row came from the extract, so wiping and rebuilding lost nothing. It stopped being harmless once the platform started minting rows Access has never heard of: allocated portal NUMids, customers created in the staff UI, OCR-captured policies, app-booked ledger rows, uploaded documents. REIMPORT is a button in /operaciones, so that was one click away. native_guard.py counts what only exists here and exits 3; run_all.py runs it before the first truncate and stops. Detecting an allocated NUMid needs the staged Parquet -- the customer holds an ordinary-looking (utilities, DATGRAL, '1172') ref, so "customer has no refs" cannot see it and only comparing against the extract can. Missing staging is therefore treated as blocking rather than as "nothing to protect". The guard does not teach full mode to preserve anything: --sync already upserts legacy rows against the existing refs and leaves the rest alone, and rebuilding that inside full mode would re-implement it. --force-full (checkbox in the REIMPORT confirm, recorded in the audit log) deletes them deliberately. Verified against dev: clean before, exit 3 listing utilities/1172 with a synthetic ref present, clean again after removing it. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -33,7 +33,9 @@ export const UTILITIES_TABLE = "DATGRAL";
|
||||
* insurance/DATGRAL is a SEPARATE id space that reuses the same sourceTable name
|
||||
* and runs past 4,000. It must never be read as a NUMid, and never allocated
|
||||
* from: the portal cannot resolve those ids. Every query here filters on BOTH
|
||||
* columns for that reason, never on sourceTable alone.
|
||||
* columns for that reason, never on sourceTable alone. A customer can also hold
|
||||
* more than one insurance ref — 16 of them do, where several insurance rows
|
||||
* folded into one customer — so those are tested with EXISTS rather than joined.
|
||||
*/
|
||||
const POOL = {
|
||||
sourceSystem: UTILITIES_SYSTEM,
|
||||
|
||||
@@ -117,11 +117,17 @@ export class OpsController {
|
||||
@Post("jobs")
|
||||
async startJob(@Body() dto: StartJobDto, @Req() req: Request) {
|
||||
const userId = this.actingId(req);
|
||||
const job = await this.ops.startJob(dto.kind, { file: dto.file }, userId);
|
||||
const job = await this.ops.startJob(
|
||||
dto.kind,
|
||||
{ file: dto.file, forceFull: dto.forceFull },
|
||||
userId,
|
||||
);
|
||||
void this.audit.log(userId, "ops.job.start", {
|
||||
jobId: job.id,
|
||||
kind: dto.kind,
|
||||
file: dto.file,
|
||||
// Recorded because this is the flag that authorised deleting native rows.
|
||||
forceFull: dto.forceFull,
|
||||
});
|
||||
return job;
|
||||
}
|
||||
|
||||
@@ -415,12 +415,19 @@ export class OpsService implements OnModuleInit {
|
||||
const out = shq(path.join(this.backupDir, file));
|
||||
const py = await this.pythonBin();
|
||||
const runAll = shq(path.join(this.migrationDir, "run_all.py"));
|
||||
// run_all.py runs native_guard.py before it truncates anything and exits
|
||||
// without touching the database when the target holds rows that only
|
||||
// exist here — allocated portal NUMids, app-created customers, OCR
|
||||
// captures. --force-full is what the operator ticks to delete them
|
||||
// anyway; without it the job fails with the list.
|
||||
const force = params.forceFull === true;
|
||||
const cmd =
|
||||
`${PIPEFAIL}echo '== Respaldo de seguridad previo ==' && ` +
|
||||
`${this.dumpCommand(flags, db, out)} && ` +
|
||||
`echo '== Reimportación desde carpeta de ingesta ==' && ` +
|
||||
`${shq(py)} ${runAll} --env ${shq(this.migrationEnv)} --stage`;
|
||||
return { cmd, resolvedParams: { safetyBackup: file } };
|
||||
`${shq(py)} ${runAll} --env ${shq(this.migrationEnv)} --stage` +
|
||||
(force ? " --force-full" : "");
|
||||
return { cmd, resolvedParams: { safetyBackup: file, forceFull: force } };
|
||||
}
|
||||
|
||||
throw new BadRequestException(`Operación no soportada: ${kind}`);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { IsEnum, IsOptional, IsString } from "class-validator";
|
||||
import { IsBoolean, IsEnum, IsOptional, IsString } from "class-validator";
|
||||
import { OpsJobKind } from "@jorgecuadros/database";
|
||||
|
||||
export class StartJobDto {
|
||||
@@ -9,4 +9,13 @@ export class StartJobDto {
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
file?: string;
|
||||
|
||||
/**
|
||||
* REIMPORT only: proceed even though the rebuild deletes rows that exist only
|
||||
* in the platform. Off by default, so the guard in run_all.py stops the job
|
||||
* and lists what would be lost rather than the operator finding out after.
|
||||
*/
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
forceFull?: boolean;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user