fix(migration): make Phase B additive sync actually work + verify end-to-end
The --sync path had never been run and was broken in several ways. Fixed and verified against the dev DB (two consecutive syncs, both exit 0, 32/32 assertions: stable PKs, manual-row preservation, changed-row updates, legacy-delete, no child duplication, zero FK orphans; idempotent). - policies/properties: reuse each legacy row's existing id (by provenance) BEFORE building child rows, so children no longer point at a discarded fresh uuid; rebuild legacy-owned children via scoped delete + reinsert. - customers: replace zip(customers, refs) (mispaired almost every row) with a ref-grouped id remap; names now restore and no spurious customers appear. - drop the invalid Vehicle @@unique(legacySourceTable, legacyId) — one legacy policy row carries up to 3 vehicles sharing a legacyId; handle via delete+reinsert. - upsert lookup tables (policy_types, insurance_providers, type_transactions, adjusters) by natural name and remap child FKs instead of inserting fresh uuids that nothing points at. - transactions: drop updatedAt=NOW() (no such column); guard report formatting on NULL legacySourceTable (manual rows). Same report guard in bank. - add manual-safe prune (prune_empty_customers.py --sync, in SYNC_STEPS): prune only legacy-owned empties, never manually-added customers. web: customer-detail mini tx list now strikes voided rows with an "(anulado)" tag (was the last void-UI rendering gap; /estado-cuenta already handled it). docs: RESUME.md updated — Phase B sync marked verified end-to-end, void-UI browser pass recorded. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -37,7 +37,7 @@ from pathlib import Path
|
||||
import pandas as pd
|
||||
|
||||
from dbenv import connect, env_arg
|
||||
from sync import parse_mode
|
||||
from sync import parse_mode, existing_ids, delete_missing
|
||||
|
||||
STG = Path(__file__).parent / "output" / "stg_seguros"
|
||||
LEGACY_DB = "SEGUROS 16_be"
|
||||
@@ -180,6 +180,14 @@ def main():
|
||||
c.execute("SELECT legacyId, customerId FROM customer_legacy_refs WHERE sourceSystem='insurance'")
|
||||
cust = {r[0]: r[1] for r in c.fetchall()}
|
||||
|
||||
# Sync mode reuses each legacy policy's existing id (keyed by provenance) so
|
||||
# its PK is stable and every child row built below points at the right
|
||||
# parent. New legacy policies fall through to a fresh uuid.
|
||||
existing_pol = existing_ids(
|
||||
c, "policies", ("legacySourceDb", "legacySourceTable", "legacyId"),
|
||||
"WHERE legacyId IS NOT NULL") if sync_mode else {}
|
||||
pol_keys: set = set()
|
||||
|
||||
policies, insts, vehicles, drivers = [], [], [], []
|
||||
polno_to_id = {} # policy number -> a policyId (for BENEF/DATOS linking)
|
||||
providers, ptypes = set(), set()
|
||||
@@ -198,7 +206,10 @@ def main():
|
||||
if not cid:
|
||||
skipped += 1
|
||||
continue
|
||||
pid = str(uuid.uuid4())
|
||||
legacy_pid = str(int(row["_row_num"]))
|
||||
pkey = (LEGACY_DB, table, legacy_pid)
|
||||
pol_keys.add(pkey)
|
||||
pid = existing_pol.get(pkey) or str(uuid.uuid4())
|
||||
comp = s(row.get(F.get("comp", ""), None)) if F.get("comp") else None
|
||||
if comp:
|
||||
providers.add(comp)
|
||||
@@ -316,20 +327,60 @@ def main():
|
||||
dt(r["fecha_cheque"]), s(r["num_cheque"]),
|
||||
1 if truthy(r["concluido"]) else 0, s(r["resolucion"])))
|
||||
|
||||
pol_cols = ("id,policyNumber,customerId,policyTypeId,insuranceProviderId,agentName,policyDate,"
|
||||
"policyFrom,policyTo,netPremium,policyFee,commission,total,currency,observations,"
|
||||
"coveragesJson,liquidated,liquidationNumber,liquidationDate,legacySourceDb,"
|
||||
"legacySourceTable,legacyId,updatedAt")
|
||||
ph = ",".join(["%s"] * 23)
|
||||
pol_upsert = (
|
||||
f"INSERT INTO policies ({pol_cols}) VALUES ({ph}) ON DUPLICATE KEY UPDATE "
|
||||
"customerId=VALUES(customerId),policyNumber=VALUES(policyNumber),policyTypeId=VALUES(policyTypeId),"
|
||||
"insuranceProviderId=VALUES(insuranceProviderId),agentName=VALUES(agentName),policyDate=VALUES(policyDate),"
|
||||
"policyFrom=VALUES(policyFrom),policyTo=VALUES(policyTo),netPremium=VALUES(netPremium),policyFee=VALUES(policyFee),"
|
||||
"commission=VALUES(commission),total=VALUES(total),currency=VALUES(currency),observations=VALUES(observations),"
|
||||
"coveragesJson=VALUES(coveragesJson),liquidated=VALUES(liquidated),liquidationNumber=VALUES(liquidationNumber),"
|
||||
"liquidationDate=VALUES(liquidationDate),updatedAt=VALUES(updatedAt),archivedAt=NULL")
|
||||
|
||||
if sync_mode:
|
||||
c.execute("SELECT id,name FROM policy_types")
|
||||
# policy_types / providers: upsert by their name unique, keep ids stable.
|
||||
c.execute("SELECT name,id FROM policy_types")
|
||||
ptype_ids = dict(c.fetchall())
|
||||
for n in ptypes:
|
||||
ptype_ids.setdefault(n, str(uuid.uuid4()))
|
||||
c.executemany("INSERT INTO policy_types (id,name) VALUES (%s,%s) ON DUPLICATE KEY UPDATE name=VALUES(name)", [(i, n) for n, i in ptype_ids.items()])
|
||||
c.execute("SELECT id,name FROM insurance_providers")
|
||||
c.execute("SELECT name,id FROM insurance_providers")
|
||||
prov_ids = dict(c.fetchall())
|
||||
for n in providers:
|
||||
prov_ids.setdefault(n, str(uuid.uuid4()))
|
||||
c.executemany("INSERT INTO insurance_providers (id,name) VALUES (%s,%s) ON DUPLICATE KEY UPDATE name=VALUES(name)", [(i, n) for n, i in prov_ids.items()])
|
||||
for p in policies:
|
||||
p = list(p); p[3] = ptype_ids[p[3]]; p[4] = prov_ids.get(p[4])
|
||||
c.execute(f"INSERT INTO policies ({pol_cols}) VALUES ({','.join(['%s'] * 23)}) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),policyNumber=VALUES(policyNumber),policyTypeId=VALUES(policyTypeId),insuranceProviderId=VALUES(insuranceProviderId),agentName=VALUES(agentName),policyDate=VALUES(policyDate),policyFrom=VALUES(policyFrom),policyTo=VALUES(policyTo),netPremium=VALUES(netPremium),policyFee=VALUES(policyFee),commission=VALUES(commission),total=VALUES(total),currency=VALUES(currency),observations=VALUES(observations),coveragesJson=VALUES(coveragesJson),liquidated=VALUES(liquidated),liquidationNumber=VALUES(liquidationNumber),liquidationDate=VALUES(liquidationDate),updatedAt=VALUES(updatedAt),archivedAt=NULL", tuple(p))
|
||||
|
||||
# Adjusters carry no provenance key, so they can't be upserted by one.
|
||||
# Resolve claims against the adjusters already in the DB (manual + prior
|
||||
# loads), inserting only names not present yet — keeps manual adjusters
|
||||
# and every claim's adjusterId FK valid.
|
||||
c.execute("SELECT id,name FROM adjusters")
|
||||
db_adj = {(nm or "").upper(): i for i, nm in c.fetchall()}
|
||||
adj_id_name = {aid: (nm or "").upper() for aid, comp, city, nm, tel, bp in adj_rows}
|
||||
new_adj = []
|
||||
for aid, comp, city, nm, tel, bp in adj_rows:
|
||||
if nm and nm.upper() not in db_adj:
|
||||
db_adj[nm.upper()] = aid
|
||||
new_adj.append((aid, comp, city, nm, tel, bp))
|
||||
if new_adj:
|
||||
c.executemany("INSERT INTO adjusters (id,company,city,name,phone,beeper) VALUES (%s,%s,%s,%s,%s,%s)", new_adj)
|
||||
claims = [(cl[0], cl[1], *cl[2:6], db_adj.get(adj_id_name.get(cl[6])) if cl[6] else None, *cl[7:]) for cl in claims]
|
||||
|
||||
# Rebuild every child of a legacy-owned policy before re-inserting the
|
||||
# children below (manual rows survive: vehicles by their own null
|
||||
# provenance, the rest by their parent policy's null provenance).
|
||||
c.execute("DELETE FROM vehicles WHERE legacyId IS NOT NULL")
|
||||
for tbl in ("policy_payment_installments", "insured_drivers", "policy_beneficiaries", "claims"):
|
||||
c.execute(f"DELETE ch FROM {tbl} ch JOIN policies p ON p.id=ch.policyId WHERE p.legacyId IS NOT NULL")
|
||||
|
||||
pol_rows = [tuple([p[0], p[1], p[2], ptype_ids.get(p[3]), prov_ids.get(p[4]), *p[5:]]) for p in policies]
|
||||
c.executemany(pol_upsert, pol_rows)
|
||||
# Drop legacy policies that vanished from source (children already gone).
|
||||
delete_missing(c, "policies", ("legacySourceDb", "legacySourceTable", "legacyId"), pol_keys, "WHERE legacyId IS NOT NULL")
|
||||
else:
|
||||
c.execute("SET FOREIGN_KEY_CHECKS=0")
|
||||
for t in ("policy_payment_installments", "vehicles", "insured_drivers",
|
||||
@@ -342,22 +393,8 @@ def main():
|
||||
prov_ids = {n: str(uuid.uuid4()) for n in providers}
|
||||
c.executemany("INSERT INTO insurance_providers (id,name) VALUES (%s,%s)", [(i, n) for n, i in prov_ids.items()])
|
||||
c.executemany("INSERT INTO adjusters (id,company,city,name,phone,beeper) VALUES (%s,%s,%s,%s,%s,%s)", adj_rows)
|
||||
|
||||
pol_cols = ("id,policyNumber,customerId,policyTypeId,insuranceProviderId,agentName,policyDate,"
|
||||
"policyFrom,policyTo,netPremium,policyFee,commission,total,currency,observations,"
|
||||
"coveragesJson,liquidated,liquidationNumber,liquidationDate,legacySourceDb,"
|
||||
"legacySourceTable,legacyId,updatedAt")
|
||||
if not sync_mode:
|
||||
fixed = []
|
||||
for p in policies:
|
||||
p = list(p)
|
||||
p[3] = ptype_ids.get(p[3])
|
||||
p[4] = prov_ids.get(p[4])
|
||||
fixed.append(tuple(p))
|
||||
ph = ",".join(["%s"] * 23)
|
||||
c.executemany(f"INSERT INTO policies ({pol_cols}) VALUES ({ph})", fixed)
|
||||
else:
|
||||
c.executemany("INSERT INTO policies (id,policyNumber,customerId,policyTypeId,insuranceProviderId,agentName,policyDate,policyFrom,policyTo,netPremium,policyFee,commission,total,currency,observations,coveragesJson,liquidated,liquidationNumber,liquidationDate,legacySourceDb,legacySourceTable,legacyId,updatedAt) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),policyNumber=VALUES(policyNumber),policyTypeId=VALUES(policyTypeId),insuranceProviderId=VALUES(insuranceProviderId),agentName=VALUES(agentName),policyDate=VALUES(policyDate),policyFrom=VALUES(policyFrom),policyTo=VALUES(policyTo),netPremium=VALUES(netPremium),policyFee=VALUES(policyFee),commission=VALUES(commission),total=VALUES(total),currency=VALUES(currency),observations=VALUES(observations),coveragesJson=VALUES(coveragesJson),liquidated=VALUES(liquidated),liquidationNumber=VALUES(liquidationNumber),liquidationDate=VALUES(liquidationDate),updatedAt=VALUES(updatedAt),archivedAt=NULL", [tuple([p[0],p[1],p[2],ptype_ids.get(p[3]),prov_ids.get(p[4]),*p[5:]]) for p in policies])
|
||||
pol_rows = [tuple([p[0], p[1], p[2], ptype_ids.get(p[3]), prov_ids.get(p[4]), *p[5:]]) for p in policies]
|
||||
c.executemany(f"INSERT INTO policies ({pol_cols}) VALUES ({ph})", pol_rows)
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user