diff --git a/migration/transform_customers.py b/migration/transform_customers.py index 3fdb74b..15dfb3e 100644 --- a/migration/transform_customers.py +++ b/migration/transform_customers.py @@ -189,6 +189,11 @@ def customer_from_utilities(row, name_index) -> dict: customerSince=as_date(row["cliente_desde"]), status=as_bool(row["status"]), feeAmount=as_decimal(row["fee"]), + # DATGRAL.TIPO is the minimum-balance threshold (100/200/300/500 — + # 1,017 of 1,172 customers carry one), NOT an identification or account + # type as the column name suggests. It reaches the website as + # datosfreak.TIPO and is returned to the customer app as `minBalance`. + minimumBalance=as_decimal(row["tipo"]), updatedAt=NOW, ) @@ -217,6 +222,7 @@ def customer_from_insurance(row, name_index) -> dict: customerSince=None, status=1, feeAmount=None, + minimumBalance=None, updatedAt=NOW, ) @@ -225,7 +231,7 @@ _CUST_COLS = [ "id", "name", "nameSource", "nameMissing", "addressLine1", "addressLine2", "city", "state", "zipCode", "country", "phone", "mobile", "fax", "email", "notes", "identificationType", "identificationNumber", "identificationExpiration", "customerSince", - "status", "feeAmount", "updatedAt", + "status", "feeAmount", "minimumBalance", "updatedAt", ] @@ -316,7 +322,7 @@ def main() -> None: remap[rec["id"]] = stable or rec["id"] for rec in customers: rec["id"] = remap[rec["id"]] - cur.execute(f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE name=VALUES(name),nameSource=VALUES(nameSource),nameMissing=VALUES(nameMissing),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),city=VALUES(city),state=VALUES(state),zipCode=VALUES(zipCode),country=VALUES(country),phone=VALUES(phone),mobile=VALUES(mobile),fax=VALUES(fax),email=VALUES(email),notes=VALUES(notes),identificationType=VALUES(identificationType),identificationNumber=VALUES(identificationNumber),identificationExpiration=VALUES(identificationExpiration),customerSince=VALUES(customerSince),status=VALUES(status),feeAmount=VALUES(feeAmount),updatedAt=VALUES(updatedAt)", tuple(rec[c] for c in _CUST_COLS)) + cur.execute(f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE name=VALUES(name),nameSource=VALUES(nameSource),nameMissing=VALUES(nameMissing),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),city=VALUES(city),state=VALUES(state),zipCode=VALUES(zipCode),country=VALUES(country),phone=VALUES(phone),mobile=VALUES(mobile),fax=VALUES(fax),email=VALUES(email),notes=VALUES(notes),identificationType=VALUES(identificationType),identificationNumber=VALUES(identificationNumber),identificationExpiration=VALUES(identificationExpiration),customerSince=VALUES(customerSince),status=VALUES(status),feeAmount=VALUES(feeAmount),minimumBalance=VALUES(minimumBalance),updatedAt=VALUES(updatedAt)", tuple(rec[c] for c in _CUST_COLS)) for ref in refs: cur.execute("INSERT INTO customer_legacy_refs (id,customerId,sourceSystem,sourceTable,legacyId) VALUES (%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId)", (ref[0], remap[ref[1]], ref[2], ref[3], ref[4])) diff --git a/migration/transform_transactions.py b/migration/transform_transactions.py index 4ba8c03..b810412 100644 --- a/migration/transform_transactions.py +++ b/migration/transform_transactions.py @@ -112,6 +112,29 @@ def main(): type_rows.append((tid, en, s(r["espa_ol"]), 0)) type_map[en.upper()] = tid + def type_id_for(raw) -> str | None: + """Resolve a transaction type, minting one when the lookup lacks it. + + The Access `TYPE OF TRX` table is a stale pick-list, not a constraint — + staff free-text straight into DATOS2, so 78 values covering 3,939 rows + (BALANCE FORWARD 1,188, ANNUAL FEE 1,116, IZZI 367, ...) appear in the + ledger but not the lookup. Leaving those unmapped stored typeId NULL and + lost the label outright: nothing else on `transactions` carries the type + text, so the row rendered blank and was unrecoverable after migration. + Minting from the literal keeps the display string; nameEs stays NULL + because only the lookup has translations. + """ + en = s(raw) + if not en: + return None + key = en.upper() + tid = type_map.get(key) + if tid is None: + tid = str(uuid.uuid4()) + type_rows.append((tid, en, None, 0)) + type_map[key] = tid + return tid + xr = load("stg_utilities", "tipo_hist") xr_rows = [] for _, r in xr.iterrows(): @@ -193,7 +216,7 @@ def main(): td = dt(r["date"]) if td is None: skip_date += 1; continue - tid = type_map.get((s(r["type_of_trx"]) or "").upper()) + tid = type_id_for(r["type_of_trx"]) add(cid, "UTILITY", td, dec(r["chargecredit"], Decimal(0)), "MXN", period=s(r["period"]), reference=s(r["refer"]), typeid=tid, check=s(r["cheque"]), src_db="UTILITIES", src_tbl=legacy_tbl,