From 7226772c22a22a16b19fa39e0f96ed8602402044 Mon Sep 17 00:00:00 2001 From: Ricardo Mancinas Date: Mon, 3 Aug 2026 12:29:49 -0700 Subject: [PATCH] fix(migration): recover transaction type labels and minimum balance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fields the customer-facing site reads were being dropped on the way in from Access. transform_transactions.py mapped DATOS2's type string through the Access `TYPE OF TRX` table and stored NULL on a miss. That table is a stale pick-list rather than a constraint — staff free-text straight into DATOS2 — so 78 distinct values covering 3,939 rows never resolved, including BALANCE FORWARD (1,188) and ANNUAL FEE (1,116). Nothing else on `transactions` carries the type text, so those rows lost their label outright and rendered blank. Now mints a type_transactions row from the literal string when the lookup lacks it; nameEs stays NULL since only the lookup has translations. transform_customers.py never carried DATGRAL.TIPO, leaving customers.minimumBalance empty on every row despite the column existing. TIPO is the minimum-balance threshold (100/200/300/500; 1,017 of 1,172 customers carry one), not an account type as the name suggests — the customer app shows it as `minBalance`. Added to the insert list and to the ON DUPLICATE KEY UPDATE clause, without which --sync would silently skip it on existing rows. Both land on the next `run_all.py --sync` reload. Co-Authored-By: Claude Opus 5 --- migration/transform_customers.py | 10 ++++++++-- migration/transform_transactions.py | 25 ++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/migration/transform_customers.py b/migration/transform_customers.py index 3fdb74b..15dfb3e 100644 --- a/migration/transform_customers.py +++ b/migration/transform_customers.py @@ -189,6 +189,11 @@ def customer_from_utilities(row, name_index) -> dict: customerSince=as_date(row["cliente_desde"]), status=as_bool(row["status"]), feeAmount=as_decimal(row["fee"]), + # DATGRAL.TIPO is the minimum-balance threshold (100/200/300/500 — + # 1,017 of 1,172 customers carry one), NOT an identification or account + # type as the column name suggests. It reaches the website as + # datosfreak.TIPO and is returned to the customer app as `minBalance`. + minimumBalance=as_decimal(row["tipo"]), updatedAt=NOW, ) @@ -217,6 +222,7 @@ def customer_from_insurance(row, name_index) -> dict: customerSince=None, status=1, feeAmount=None, + minimumBalance=None, updatedAt=NOW, ) @@ -225,7 +231,7 @@ _CUST_COLS = [ "id", "name", "nameSource", "nameMissing", "addressLine1", "addressLine2", "city", "state", "zipCode", "country", "phone", "mobile", "fax", "email", "notes", "identificationType", "identificationNumber", "identificationExpiration", "customerSince", - "status", "feeAmount", "updatedAt", + "status", "feeAmount", "minimumBalance", "updatedAt", ] @@ -316,7 +322,7 @@ def main() -> None: remap[rec["id"]] = stable or rec["id"] for rec in customers: rec["id"] = remap[rec["id"]] - cur.execute(f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE name=VALUES(name),nameSource=VALUES(nameSource),nameMissing=VALUES(nameMissing),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),city=VALUES(city),state=VALUES(state),zipCode=VALUES(zipCode),country=VALUES(country),phone=VALUES(phone),mobile=VALUES(mobile),fax=VALUES(fax),email=VALUES(email),notes=VALUES(notes),identificationType=VALUES(identificationType),identificationNumber=VALUES(identificationNumber),identificationExpiration=VALUES(identificationExpiration),customerSince=VALUES(customerSince),status=VALUES(status),feeAmount=VALUES(feeAmount),updatedAt=VALUES(updatedAt)", tuple(rec[c] for c in _CUST_COLS)) + cur.execute(f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE name=VALUES(name),nameSource=VALUES(nameSource),nameMissing=VALUES(nameMissing),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),city=VALUES(city),state=VALUES(state),zipCode=VALUES(zipCode),country=VALUES(country),phone=VALUES(phone),mobile=VALUES(mobile),fax=VALUES(fax),email=VALUES(email),notes=VALUES(notes),identificationType=VALUES(identificationType),identificationNumber=VALUES(identificationNumber),identificationExpiration=VALUES(identificationExpiration),customerSince=VALUES(customerSince),status=VALUES(status),feeAmount=VALUES(feeAmount),minimumBalance=VALUES(minimumBalance),updatedAt=VALUES(updatedAt)", tuple(rec[c] for c in _CUST_COLS)) for ref in refs: cur.execute("INSERT INTO customer_legacy_refs (id,customerId,sourceSystem,sourceTable,legacyId) VALUES (%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId)", (ref[0], remap[ref[1]], ref[2], ref[3], ref[4])) diff --git a/migration/transform_transactions.py b/migration/transform_transactions.py index 4ba8c03..b810412 100644 --- a/migration/transform_transactions.py +++ b/migration/transform_transactions.py @@ -112,6 +112,29 @@ def main(): type_rows.append((tid, en, s(r["espa_ol"]), 0)) type_map[en.upper()] = tid + def type_id_for(raw) -> str | None: + """Resolve a transaction type, minting one when the lookup lacks it. + + The Access `TYPE OF TRX` table is a stale pick-list, not a constraint — + staff free-text straight into DATOS2, so 78 values covering 3,939 rows + (BALANCE FORWARD 1,188, ANNUAL FEE 1,116, IZZI 367, ...) appear in the + ledger but not the lookup. Leaving those unmapped stored typeId NULL and + lost the label outright: nothing else on `transactions` carries the type + text, so the row rendered blank and was unrecoverable after migration. + Minting from the literal keeps the display string; nameEs stays NULL + because only the lookup has translations. + """ + en = s(raw) + if not en: + return None + key = en.upper() + tid = type_map.get(key) + if tid is None: + tid = str(uuid.uuid4()) + type_rows.append((tid, en, None, 0)) + type_map[key] = tid + return tid + xr = load("stg_utilities", "tipo_hist") xr_rows = [] for _, r in xr.iterrows(): @@ -193,7 +216,7 @@ def main(): td = dt(r["date"]) if td is None: skip_date += 1; continue - tid = type_map.get((s(r["type_of_trx"]) or "").upper()) + tid = type_id_for(r["type_of_trx"]) add(cid, "UTILITY", td, dec(r["chargecredit"], Decimal(0)), "MXN", period=s(r["period"]), reference=s(r["refer"]), typeid=tid, check=s(r["cheque"]), src_db="UTILITIES", src_tbl=legacy_tbl,