fix(migration): make Phase B additive sync actually work + verify end-to-end
Build and Push Images / Build jorgecuadros-web (push) Successful in 1m37s
Build and Push Images / Build jorgecuadros-api (push) Successful in 2m9s

The --sync path had never been run and was broken in several ways. Fixed and
verified against the dev DB (two consecutive syncs, both exit 0, 32/32
assertions: stable PKs, manual-row preservation, changed-row updates,
legacy-delete, no child duplication, zero FK orphans; idempotent).

- policies/properties: reuse each legacy row's existing id (by provenance)
  BEFORE building child rows, so children no longer point at a discarded fresh
  uuid; rebuild legacy-owned children via scoped delete + reinsert.
- customers: replace zip(customers, refs) (mispaired almost every row) with a
  ref-grouped id remap; names now restore and no spurious customers appear.
- drop the invalid Vehicle @@unique(legacySourceTable, legacyId) — one legacy
  policy row carries up to 3 vehicles sharing a legacyId; handle via delete+reinsert.
- upsert lookup tables (policy_types, insurance_providers, type_transactions,
  adjusters) by natural name and remap child FKs instead of inserting fresh
  uuids that nothing points at.
- transactions: drop updatedAt=NOW() (no such column); guard report formatting
  on NULL legacySourceTable (manual rows). Same report guard in bank.
- add manual-safe prune (prune_empty_customers.py --sync, in SYNC_STEPS): prune
  only legacy-owned empties, never manually-added customers.

web: customer-detail mini tx list now strikes voided rows with an "(anulado)"
tag (was the last void-UI rendering gap; /estado-cuenta already handled it).

docs: RESUME.md updated — Phase B sync marked verified end-to-end, void-UI
browser pass recorded.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-24 13:37:22 -07:00
co-authored by Claude Opus 4.8
parent 8802f08d4f
commit 1b79b43a54
11 changed files with 211 additions and 76 deletions
+15 -10
View File
@@ -112,6 +112,12 @@ def main():
"WHERE sourceSystem='utilities' AND sourceTable='DATGRAL'")
cust_map = {r[0]: r[1] for r in cur.fetchall()}
# Sync reuses each legacy property's existing id (keyed by provenance) so its
# PK is stable AND the services/trust rows built below point at the right
# parent. New legacy rows fall through to a fresh uuid.
existing_prop = existing_ids(cur, "properties", ("legacySourceTable", "legacyId"),
"WHERE legacyId IS NOT NULL") if sync_mode else {}
dm = load("datmex")
pf = load("profile")
# PROFILE flags by join key (best-effort; key nearly unique in PROFILE)
@@ -133,7 +139,7 @@ def main():
legacy_id = str(int(row["_row_num"]))
prop_keys.add(("DATMEX", legacy_id))
pid = str(uuid.uuid4())
pid = existing_prop.get(("DATMEX", legacy_id)) or str(uuid.uuid4())
addr2_parts = []
for lbl, col in (("CASA", "casa"), ("MZ", "manzana"), ("LOTE", "lote")):
if s_keep0(row[col]):
@@ -206,16 +212,14 @@ def main():
# Fresh rebuild (children first), or additive upsert for legacy-owned rows.
if sync_mode:
existing = existing_ids(cur, "properties", ("legacySourceTable", "legacyId"), "WHERE legacyId IS NOT NULL")
for row in props:
key = (row[8], row[9])
if key in existing:
row = list(row); row[0] = existing[key]
cur.execute(
"INSERT INTO properties (id,customerId,addressLine1,addressLine2,phone1,phone2,phone3,zone,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),phone1=VALUES(phone1),phone2=VALUES(phone2),phone3=VALUES(phone3),zone=VALUES(zone),archivedAt=NULL", tuple(row))
delete_missing(cur, "properties", ("legacySourceTable", "legacyId"), prop_keys, "WHERE legacyId IS NOT NULL")
# Children first (scoped to legacy-owned rows so manual rows survive),
# then upsert properties (ids already stable), then drop legacy rows
# gone from source, then re-insert the rebuilt children.
cur.execute("DELETE ps FROM property_services ps JOIN properties p ON p.id=ps.propertyId WHERE p.legacyId IS NOT NULL")
cur.execute("DELETE ta FROM trust_accounts ta JOIN properties p ON p.id=ta.propertyId WHERE p.legacyId IS NOT NULL")
cur.executemany(
"INSERT INTO properties (id,customerId,addressLine1,addressLine2,phone1,phone2,phone3,zone,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),phone1=VALUES(phone1),phone2=VALUES(phone2),phone3=VALUES(phone3),zone=VALUES(zone),archivedAt=NULL", props)
delete_missing(cur, "properties", ("legacySourceTable", "legacyId"), prop_keys, "WHERE legacyId IS NOT NULL")
cur.executemany("INSERT INTO property_services (id,propertyId,kind,accountNumber,meterNumber,route,dueDay,active,notes) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", services)
cur.executemany("INSERT INTO trust_accounts (id,propertyId,bankName,trustNumber,bankFee,dueDate1,dueDate2) VALUES (%s,%s,%s,%s,%s,%s,%s)", trusts)
else:
@@ -246,7 +250,8 @@ def main():
print(f" {k:14} {n}")
print(f" -> trust_accounts : {n_t}")
print(f" orphan properties (bad customer FK): {orphans}")
assert n_p == len(props) and orphans == 0, "property load invariant failed"
# n_p == len(props) is a full-load invariant; sync keeps manual rows too.
assert (sync_mode or n_p == len(props)) and orphans == 0, "property load invariant failed"
print(" validation: OK")
conn.close()