fix(migration): make Phase B additive sync actually work + verify end-to-end
The --sync path had never been run and was broken in several ways. Fixed and verified against the dev DB (two consecutive syncs, both exit 0, 32/32 assertions: stable PKs, manual-row preservation, changed-row updates, legacy-delete, no child duplication, zero FK orphans; idempotent). - policies/properties: reuse each legacy row's existing id (by provenance) BEFORE building child rows, so children no longer point at a discarded fresh uuid; rebuild legacy-owned children via scoped delete + reinsert. - customers: replace zip(customers, refs) (mispaired almost every row) with a ref-grouped id remap; names now restore and no spurious customers appear. - drop the invalid Vehicle @@unique(legacySourceTable, legacyId) — one legacy policy row carries up to 3 vehicles sharing a legacyId; handle via delete+reinsert. - upsert lookup tables (policy_types, insurance_providers, type_transactions, adjusters) by natural name and remap child FKs instead of inserting fresh uuids that nothing points at. - transactions: drop updatedAt=NOW() (no such column); guard report formatting on NULL legacySourceTable (manual rows). Same report guard in bank. - add manual-safe prune (prune_empty_customers.py --sync, in SYNC_STEPS): prune only legacy-owned empties, never manually-added customers. web: customer-detail mini tx list now strikes voided rows with an "(anulado)" tag (was the last void-UI rendering gap; /estado-cuenta already handled it). docs: RESUME.md updated — Phase B sync marked verified end-to-end, void-UI browser pass recorded. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -112,6 +112,12 @@ def main():
|
||||
"WHERE sourceSystem='utilities' AND sourceTable='DATGRAL'")
|
||||
cust_map = {r[0]: r[1] for r in cur.fetchall()}
|
||||
|
||||
# Sync reuses each legacy property's existing id (keyed by provenance) so its
|
||||
# PK is stable AND the services/trust rows built below point at the right
|
||||
# parent. New legacy rows fall through to a fresh uuid.
|
||||
existing_prop = existing_ids(cur, "properties", ("legacySourceTable", "legacyId"),
|
||||
"WHERE legacyId IS NOT NULL") if sync_mode else {}
|
||||
|
||||
dm = load("datmex")
|
||||
pf = load("profile")
|
||||
# PROFILE flags by join key (best-effort; key nearly unique in PROFILE)
|
||||
@@ -133,7 +139,7 @@ def main():
|
||||
|
||||
legacy_id = str(int(row["_row_num"]))
|
||||
prop_keys.add(("DATMEX", legacy_id))
|
||||
pid = str(uuid.uuid4())
|
||||
pid = existing_prop.get(("DATMEX", legacy_id)) or str(uuid.uuid4())
|
||||
addr2_parts = []
|
||||
for lbl, col in (("CASA", "casa"), ("MZ", "manzana"), ("LOTE", "lote")):
|
||||
if s_keep0(row[col]):
|
||||
@@ -206,16 +212,14 @@ def main():
|
||||
|
||||
# Fresh rebuild (children first), or additive upsert for legacy-owned rows.
|
||||
if sync_mode:
|
||||
existing = existing_ids(cur, "properties", ("legacySourceTable", "legacyId"), "WHERE legacyId IS NOT NULL")
|
||||
for row in props:
|
||||
key = (row[8], row[9])
|
||||
if key in existing:
|
||||
row = list(row); row[0] = existing[key]
|
||||
cur.execute(
|
||||
"INSERT INTO properties (id,customerId,addressLine1,addressLine2,phone1,phone2,phone3,zone,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),phone1=VALUES(phone1),phone2=VALUES(phone2),phone3=VALUES(phone3),zone=VALUES(zone),archivedAt=NULL", tuple(row))
|
||||
delete_missing(cur, "properties", ("legacySourceTable", "legacyId"), prop_keys, "WHERE legacyId IS NOT NULL")
|
||||
# Children first (scoped to legacy-owned rows so manual rows survive),
|
||||
# then upsert properties (ids already stable), then drop legacy rows
|
||||
# gone from source, then re-insert the rebuilt children.
|
||||
cur.execute("DELETE ps FROM property_services ps JOIN properties p ON p.id=ps.propertyId WHERE p.legacyId IS NOT NULL")
|
||||
cur.execute("DELETE ta FROM trust_accounts ta JOIN properties p ON p.id=ta.propertyId WHERE p.legacyId IS NOT NULL")
|
||||
cur.executemany(
|
||||
"INSERT INTO properties (id,customerId,addressLine1,addressLine2,phone1,phone2,phone3,zone,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),phone1=VALUES(phone1),phone2=VALUES(phone2),phone3=VALUES(phone3),zone=VALUES(zone),archivedAt=NULL", props)
|
||||
delete_missing(cur, "properties", ("legacySourceTable", "legacyId"), prop_keys, "WHERE legacyId IS NOT NULL")
|
||||
cur.executemany("INSERT INTO property_services (id,propertyId,kind,accountNumber,meterNumber,route,dueDay,active,notes) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", services)
|
||||
cur.executemany("INSERT INTO trust_accounts (id,propertyId,bankName,trustNumber,bankFee,dueDate1,dueDate2) VALUES (%s,%s,%s,%s,%s,%s,%s)", trusts)
|
||||
else:
|
||||
@@ -246,7 +250,8 @@ def main():
|
||||
print(f" {k:14} {n}")
|
||||
print(f" -> trust_accounts : {n_t}")
|
||||
print(f" orphan properties (bad customer FK): {orphans}")
|
||||
assert n_p == len(props) and orphans == 0, "property load invariant failed"
|
||||
# n_p == len(props) is a full-load invariant; sync keeps manual rows too.
|
||||
assert (sync_mode or n_p == len(props)) and orphans == 0, "property load invariant failed"
|
||||
print(" validation: OK")
|
||||
conn.close()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user