fix(migration): make Phase B additive sync actually work + verify end-to-end
Build and Push Images / Build jorgecuadros-web (push) Successful in 1m37s
Build and Push Images / Build jorgecuadros-api (push) Successful in 2m9s

The --sync path had never been run and was broken in several ways. Fixed and
verified against the dev DB (two consecutive syncs, both exit 0, 32/32
assertions: stable PKs, manual-row preservation, changed-row updates,
legacy-delete, no child duplication, zero FK orphans; idempotent).

- policies/properties: reuse each legacy row's existing id (by provenance)
  BEFORE building child rows, so children no longer point at a discarded fresh
  uuid; rebuild legacy-owned children via scoped delete + reinsert.
- customers: replace zip(customers, refs) (mispaired almost every row) with a
  ref-grouped id remap; names now restore and no spurious customers appear.
- drop the invalid Vehicle @@unique(legacySourceTable, legacyId) — one legacy
  policy row carries up to 3 vehicles sharing a legacyId; handle via delete+reinsert.
- upsert lookup tables (policy_types, insurance_providers, type_transactions,
  adjusters) by natural name and remap child FKs instead of inserting fresh
  uuids that nothing points at.
- transactions: drop updatedAt=NOW() (no such column); guard report formatting
  on NULL legacySourceTable (manual rows). Same report guard in bank.
- add manual-safe prune (prune_empty_customers.py --sync, in SYNC_STEPS): prune
  only legacy-owned empties, never manually-added customers.

web: customer-detail mini tx list now strikes voided rows with an "(anulado)"
tag (was the last void-UI rendering gap; /estado-cuenta already handled it).

docs: RESUME.md updated — Phase B sync marked verified end-to-end, void-UI
browser pass recorded.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-24 13:37:22 -07:00
co-authored by Claude Opus 4.8
parent 8802f08d4f
commit 1b79b43a54
11 changed files with 211 additions and 76 deletions
+33 -12
View File
@@ -293,18 +293,33 @@ def main() -> None:
refs.append((str(uuid.uuid4()), cust_id, "insurance", "DATGRAL", ins_id))
placeholders = ",".join(["%s"] * len(_CUST_COLS))
remap: dict[str, str] = {} # in-memory customer id -> stable (DB) id
if sync_mode:
existing = {}
cur.execute("SELECT id,sourceSystem,sourceTable,legacyId,customerId FROM customer_legacy_refs")
for rid, system, table, legacy, customer_id in cur.fetchall():
existing[(system, table, legacy)] = (rid, customer_id)
for rec, ref in zip(customers, refs):
key = (ref[2], ref[3], ref[4])
customer_id = existing.get(key, (None, rec["id"]))[1]
rec["id"] = customer_id
# Resolve each in-memory customer to a stable id: if ANY of its legacy
# refs already exists in the DB, reuse that customer's id (keeps PKs
# stable and preserves manual edits). `customers` and `refs` are
# different-length, differently-ordered lists — merged identities add a
# ref without a customer — so refs are grouped by their owning customer,
# never positionally zipped (the old zip mispaired almost every row).
cur.execute("SELECT sourceSystem,sourceTable,legacyId,customerId FROM customer_legacy_refs")
ref_existing = {(sy, tb, lg): cid for sy, tb, lg, cid in cur.fetchall()}
refs_by_cust: dict[str, list] = {}
for ref in refs: # ref = (refId, custInMemId, system, table, legacyId)
refs_by_cust.setdefault(ref[1], []).append(ref)
for rec in customers:
stable = None
for ref in refs_by_cust.get(rec["id"], []):
cid = ref_existing.get((ref[2], ref[3], ref[4]))
if cid:
stable = cid
break
remap[rec["id"]] = stable or rec["id"]
for rec in customers:
rec["id"] = remap[rec["id"]]
cur.execute(f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE name=VALUES(name),nameSource=VALUES(nameSource),nameMissing=VALUES(nameMissing),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),city=VALUES(city),state=VALUES(state),zipCode=VALUES(zipCode),country=VALUES(country),phone=VALUES(phone),mobile=VALUES(mobile),fax=VALUES(fax),email=VALUES(email),notes=VALUES(notes),identificationType=VALUES(identificationType),identificationNumber=VALUES(identificationNumber),identificationExpiration=VALUES(identificationExpiration),customerSince=VALUES(customerSince),status=VALUES(status),feeAmount=VALUES(feeAmount),updatedAt=VALUES(updatedAt)", tuple(rec[c] for c in _CUST_COLS))
ref = (existing.get(key, (ref[0], customer_id))[0], customer_id, *ref[2:])
cur.execute("INSERT INTO customer_legacy_refs (id,customerId,sourceSystem,sourceTable,legacyId) VALUES (%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId)", ref)
for ref in refs:
cur.execute("INSERT INTO customer_legacy_refs (id,customerId,sourceSystem,sourceTable,legacyId) VALUES (%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId)",
(ref[0], remap[ref[1]], ref[2], ref[3], ref[4]))
else:
cur.executemany(
f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders})",
@@ -317,7 +332,10 @@ def main() -> None:
# Enrich linked customers with insurance-only ID-doc fields, and fill any
# contact fields the utilities master left empty (COALESCE keeps master's).
# In sync mode the enrich targets carry in-memory ids, so map them to the
# stable DB ids resolved above (identity map in full mode).
for cust_id, ir in enrich:
cust_id = remap.get(cust_id, cust_id)
cur.execute(
"UPDATE customers SET "
"identificationType = COALESCE(identificationType, %s), "
@@ -366,8 +384,11 @@ def main() -> None:
print(f" from {src:16} : {n}")
print(f" of which via the linked insurance record : {from_ins_side}")
print(f" still {NO_NAME} : {still_unnamed}")
assert n_cust == len(util) + new_ins, "customer count mismatch"
assert n_refs == len(util) + len(ins), "legacy ref count mismatch"
if not sync_mode:
# Full-load invariants only: sync upserts into an already-loaded (and
# pruned) table, so these exact counts don't hold.
assert n_cust == len(util) + new_ins, "customer count mismatch"
assert n_refs == len(util) + len(ins), "legacy ref count mismatch"
print(" validation: OK")
conn.close()