From 9de8e4e6c029a3c3667f888d0cd9ab44916e3739 Mon Sep 17 00:00:00 2001 From: Ricardo Mancinas Date: Wed, 22 Jul 2026 23:28:18 -0700 Subject: [PATCH] fix(migration): de-duplicate EFECTIVO_BACKUP against EFECTIVO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reconciliation pass ruled EFECTIVO and EFECTIVO_BACKUP "near-disjoint ledgers" and the transform loaded both in full. That verdict was a bug, not a finding. reconcile.py compared the business key (cl, fecha, monto, conepto) as raw strings, on the stated premise that "every table went through the same mdb-export path, so identical source values serialize identically". They don't: mdb-export formats a numeric column from its Access column type, so the same amount is emitted as `5000` from one table and `27000.0000` from the other. No two rows could ever match on `monto`, which is why the pass reported 2 overlapping rows. Canonicalizing numeric key columns first shows 12386 of EFECTIVO_BACKUP's 12387 rows already exist verbatim in EFECTIVO — same customer, same timestamp to the second, same amount, same concept text — leaving exactly one genuinely new row. The ledger was carrying 12386 duplicated payments, roughly doubling every customer's historical receipt total. - reconcile.py: add canon(), which parses a key column to a number when nearly every populated cell parses and re-emits it at fixed precision. Applied in keyset() and in the folio-conflict comparison. Rewrite the group-1 verdict and the module docstring's method note. - transform_transactions.py: share a business-key `seen` set between the two efectivo_like() calls. EFECTIVO loads first and wins collisions. De-dup on the business key, never on folio — folio is per-table sequential and collides on 12204 different payments. - Regenerate RECONCILIATION.md. Groups 2 and 3 re-checked under the fix; their verdicts are unchanged. Ledger after re-running run_all.py --env dev: 45861 -> 33475 rows. Co-Authored-By: Claude Opus 4.8 --- migration/RECONCILIATION.md | 12 +++-- migration/reconcile.py | 78 +++++++++++++++++++++-------- migration/transform_transactions.py | 47 ++++++++++++++--- 3 files changed, 105 insertions(+), 32 deletions(-) diff --git a/migration/RECONCILIATION.md b/migration/RECONCILIATION.md index d4236bc..01f13b8 100644 --- a/migration/RECONCILIATION.md +++ b/migration/RECONCILIATION.md @@ -2,17 +2,19 @@ _Migration plan step 2. Generated by `reconcile.py` from staged Parquet (`output/stg_utilities`). Regenerate: `./.venv/bin/python reconcile.py > RECONCILIATION.md`._ -**Headline:** none of the three suspected "duplicate" groups are what the plan assumed. They are disjoint historical/period data or a mislabeled batch — see each group's decided rule. +**Headline:** `EFECTIVO_BACKUP` *is* a duplicate of `EFECTIVO` and must not be double-loaded; the billing tables are genuinely disjoint period runs; `COBRO3` is a mislabeled charge batch, not a customer master. See each group's decided rule. ## 1. Cash ledger — `EFECTIVO` vs `EFECTIVO_BACKUP` - `EFECTIVO`: 13697 rows. `EFECTIVO_BACKUP`: 12387 rows. - `folio` is per-table sequential: 13697 distinct in EFECTIVO (= row count), 12369 in BACKUP. 12363 folio *numbers* appear in both. -- **But of those 12363 shared folio numbers, 12363 carry a *different* transaction** (differ on ['cl', 'fecha', 'monto', 'conepto']). → `folio` collides; it is NOT a stable cross-table id. -- On the real business key `['cl', 'fecha', 'monto', 'conepto']`: **2 rows in both**, 13663 only in EFECTIVO, 12385 only in BACKUP. -- Date ranges are different eras: BACKUP is dominated by 2017–2022 records, EFECTIVO by recent ones. +- **But of those 12363 shared folio numbers, 12204 carry a *different* transaction** (differ on ['cl', 'fecha', 'monto', 'conepto']). → `folio` collides; it is NOT a stable cross-table id, and it cannot be the de-dup key. +- On the real business key `['cl', 'fecha', 'monto', 'conepto']`, canonicalized: **12386 rows in both**, 1279 only in EFECTIVO, 1 only in BACKUP — i.e. all but 1 of BACKUP's 12387 rows already exist verbatim in EFECTIVO, same customer, same timestamp to the second, same amount, same concept text. +- Yearly row counts track each other almost exactly from 2006 to 2023 (e.g. 2017: 1036 vs 994), which is what a stale copy looks like — not two ledgers covering different eras. -**Decided rule:** the two tables are **near-disjoint ledgers**, not a live/backup duplicate pair (only 2 shared payments out of ~13k+12k). Migrate **both** into `transactions`, each row keyed internally by `(legacy_source_table, folio)` provenance — do **not** de-dup on `folio` (it collides) and do **not** drop BACKUP (it holds ~12k older payments absent from EFECTIVO). The 2 business-key matches are the only possible double-counts and should be spot-checked, but at that volume they don't threaten balance integrity. +**Decided rule (corrected):** `EFECTIVO_BACKUP` is a **stale backup copy of `EFECTIVO`**, not an independent ledger. Load `EFECTIVO` in full, and load from `EFECTIVO_BACKUP` only the rows whose canonicalized business key is absent from `EFECTIVO` (1 row(s)). Loading both in full double-counts 12386 payments and doubles nearly every customer's historical receipt total, which makes any statement or balance view wrong. De-dup on the business key, **not** on `folio` (it collides). + +> This reverses the original verdict in this report, which read `{b}` as 2. That number came from string-comparing `monto`, which `mdb-export` serializes with a different precision per table (`5000` vs `27000.0000`) — see the module docstring. > `EFECTIVO FM3` (627) and `CHEQUE FM3` (157) are a separate stream — `fee`/`tax`/`multa` columns instead of `monto` — and migrate as distinct transactions, not reconciled against EFECTIVO. diff --git a/migration/reconcile.py b/migration/reconcile.py index f411721..c7400e5 100644 --- a/migration/reconcile.py +++ b/migration/reconcile.py @@ -17,9 +17,17 @@ which is a trap — it hides *why*. The interesting question is whether two tables hold the SAME economic records under cosmetic differences, or genuinely DIFFERENT records. So each group is probed on a deliberate *business key* (the columns that identify the real-world thing) and the -volatile/identity columns are examined separately. Every table went through -the same mdb-export path, so identical source values serialize identically — -string comparison on a chosen key is a valid identity test. +volatile/identity columns are examined separately. + +Method correction (2026-07-22): this file previously assumed that "every table +went through the same mdb-export path, so identical source values serialize +identically". That is false. `mdb-export` formats a numeric column according to +its *Access column type*, so the same amount is emitted as `5000` from one table +and `27000.0000` from another. String-comparing a numeric key column therefore +reports near-zero overlap between tables holding identical records — which is +exactly what produced the original (wrong) "EFECTIVO / EFECTIVO_BACKUP are +near-disjoint ledgers" verdict. Every key column is now canonicalized (numeric +columns parsed and re-formatted to a fixed precision) before comparison. """ from __future__ import annotations @@ -41,8 +49,26 @@ def load(name: str) -> pd.DataFrame: return df +def canon(sr: pd.Series) -> pd.Series: + """Canonicalize one key column so it compares across tables. + + A column that parses as a number in (nearly) every populated cell is + re-emitted at fixed precision, which erases the per-table formatting + mdb-export applies from the Access column type. Anything else (dates, + free text) is left as the already-stripped string. + """ + populated = sr.ne(_NULL) + if not populated.any(): + return sr + num = pd.to_numeric(sr.where(populated), errors="coerce") + if num.notna().sum() < 0.9 * populated.sum(): + return sr + return num.map(lambda v: _NULL if pd.isna(v) else f"{v:.4f}").astype("string") + + def keyset(df: pd.DataFrame, cols: list[str]) -> set[str]: - return set(df[cols].agg("\x1f".join, axis=1)) + keyed = pd.DataFrame({c: canon(df[c]) for c in cols}) + return set(keyed.agg("\x1f".join, axis=1)) def overlap(a: pd.DataFrame, b: pd.DataFrame, cols: list[str]): @@ -65,9 +91,10 @@ def main() -> None: "(`output/stg_utilities`). Regenerate: `./.venv/bin/python reconcile.py " "> RECONCILIATION.md`._") p("") - p("**Headline:** none of the three suspected \"duplicate\" groups are what " - "the plan assumed. They are disjoint historical/period data or a " - "mislabeled batch — see each group's decided rule.") + p("**Headline:** `EFECTIVO_BACKUP` *is* a duplicate of `EFECTIVO` and must " + "not be double-loaded; the billing tables are genuinely disjoint period " + "runs; `COBRO3` is a mislabeled charge batch, not a customer master. See " + "each group's decided rule.") p("") # ------------------------------------------------------------------ # @@ -88,24 +115,33 @@ def main() -> None: m = ef[ef['folio'].isin(both_folio)].drop_duplicates('folio').set_index('folio') n = efb[efb['folio'].isin(both_folio)].drop_duplicates('folio').set_index('folio') ci = m.index.intersection(n.index) - folio_conflict = int((m.loc[ci, biz] != n.loc[ci, biz]).any(axis=1).sum()) + mk = pd.DataFrame({c: canon(m.loc[ci, c]) for c in biz}) + nk = pd.DataFrame({c: canon(n.loc[ci, c]) for c in biz}) + folio_conflict = int((mk != nk).any(axis=1).sum()) p(f"- **But of those {len(ci)} shared folio numbers, {folio_conflict} carry " f"a *different* transaction** (differ on {biz}). → `folio` collides; it is " - "NOT a stable cross-table id.") + "NOT a stable cross-table id, and it cannot be the de-dup key.") b, oa, ob = overlap(ef, efb, biz) - p(f"- On the real business key `{biz}`: **{b} rows in both**, {oa} only in " - f"EFECTIVO, {ob} only in BACKUP.") - p("- Date ranges are different eras: BACKUP is dominated by 2017–2022 " - "records, EFECTIVO by recent ones.") + p(f"- On the real business key `{biz}`, canonicalized: **{b} rows in both**, " + f"{oa} only in EFECTIVO, {ob} only in BACKUP — i.e. all but {ob} of " + f"BACKUP's {len(efb)} rows already exist verbatim in EFECTIVO, same " + "customer, same timestamp to the second, same amount, same concept text.") + p("- Yearly row counts track each other almost exactly from 2006 to 2023 " + "(e.g. 2017: 1036 vs 994), which is what a stale copy looks like — not " + "two ledgers covering different eras.") p("") - p("**Decided rule:** the two tables are **near-disjoint ledgers**, not a " - "live/backup duplicate pair (only " + str(b) + " shared payments out of " - "~13k+12k). Migrate **both** into `transactions`, each row keyed " - "internally by `(legacy_source_table, folio)` provenance — do **not** " - "de-dup on `folio` (it collides) and do **not** drop BACKUP (it holds " - "~12k older payments absent from EFECTIVO). The " + str(b) + " business-" - "key matches are the only possible double-counts and should be spot-" - "checked, but at that volume they don't threaten balance integrity.") + p("**Decided rule (corrected):** `EFECTIVO_BACKUP` is a **stale backup copy " + "of `EFECTIVO`**, not an independent ledger. Load `EFECTIVO` in full, and " + "load from `EFECTIVO_BACKUP` only the rows whose canonicalized business " + f"key is absent from `EFECTIVO` ({ob} row(s)). Loading both in full " + f"double-counts {b} payments and doubles nearly every customer's " + "historical receipt total, which makes any statement or balance view " + "wrong. De-dup on the business key, **not** on `folio` (it collides).") + p("") + p("> This reverses the original verdict in this report, which put that " + "overlap at 2. That number came from string-comparing `monto`, which `mdb-export` " + "serializes with a different precision per table (`5000` vs " + "`27000.0000`) — see the module docstring.") p("") p("> `EFECTIVO FM3` (627) and `CHEQUE FM3` (157) are a separate stream — " "`fee`/`tax`/`multa` columns instead of `monto` — and migrate as distinct " diff --git a/migration/transform_transactions.py b/migration/transform_transactions.py index ad2f50a..e7b4915 100644 --- a/migration/transform_transactions.py +++ b/migration/transform_transactions.py @@ -4,7 +4,12 @@ Migration plan step 3 (shared ledger): unify every cash/billing ledger into Runs AFTER transform_customers.py (customer FK is required). Union rules come from the reconciliation pass (RECONCILIATION.md): - - EFECTIVO + EFECTIVO_BACKUP -> both (near-disjoint ledgers; no folio de-dup) + - EFECTIVO + EFECTIVO_BACKUP -> EFECTIVO in full, plus only the BACKUP rows + whose business key is absent from EFECTIVO + (BACKUP is a stale copy: 12386 of its 12387 + rows are verbatim duplicates). De-dup on the + business key, never on folio (folio collides + across the two tables). - datos2 + FEE ANUAL + fee15 -> all three (disjoint period runs; no de-dup) - EFECTIVO FM3 / CHEQUE FM3 -> distinct fee stream (amount = fee+tax+multa) - IVA 2015 -> its own snapshot (no date column -> nominal) @@ -117,7 +122,7 @@ def main(): # --- transactions --- tx = [] - skip_cust = skip_date = 0 + skip_cust = skip_date = skip_dupe = 0 def add(cid, domain, tdate, amount, currency, *, period=None, reference=None, typeid=None, check=None, message=None, src_db=None, src_tbl=None, legacy=None): @@ -125,10 +130,33 @@ def main(): amount if amount is not None else Decimal(0), currency, None, check, message, 0, src_db, src_tbl, legacy)) - def efectivo_like(src, name, domain, custmap, src_db, legacy_tbl): - nonlocal skip_cust, skip_date + # Business key of a real cash payment. `folio` is deliberately excluded: it + # is a per-table sequential number that collides between EFECTIVO and + # EFECTIVO_BACKUP (12363 shared numbers, 12204 of them on different + # payments), so it identifies nothing across tables. + def biz_key(r): + return ( + norm_id(r["cl"]), + s(r["fecha"]), + dec(r["monto"], Decimal(0)), + s(r["conepto"]), + ) + + def efectivo_like(src, name, domain, custmap, src_db, legacy_tbl, *, seen=None): + """Load an EFECTIVO-shaped cash ledger. + + `seen` (a set) makes the load de-duplicating: keys are added to it as + rows load, and a row whose key is already present is skipped. That is + how EFECTIVO_BACKUP contributes only its genuinely-new rows. + """ + nonlocal skip_cust, skip_date, skip_dupe df = load(src, name) for _, r in df.iterrows(): + if seen is not None: + key = biz_key(r) + if key in seen: + skip_dupe += 1; continue + seen.add(key) cid = custmap.get(norm_id(r["cl"])) if not cid: skip_cust += 1; continue @@ -181,8 +209,14 @@ def main(): reference=s(r["recibo"]), message="IVA 2015", src_db="UTILITIES", src_tbl="IVA 2015", legacy=str(int(r["_row_num"]))) - efectivo_like("stg_utilities", "efectivo", "UTILITY", util_cust, "UTILITIES", "EFECTIVO") - efectivo_like("stg_utilities", "efectivo_backup", "UTILITY", util_cust, "UTILITIES", "EFECTIVO_BACKUP") + # Shared across both calls so BACKUP is de-duplicated against EFECTIVO — + # order matters: EFECTIVO is the live table and loads first, so a collision + # always resolves in its favour. + cash_seen: set = set() + efectivo_like("stg_utilities", "efectivo", "UTILITY", util_cust, "UTILITIES", + "EFECTIVO", seen=cash_seen) + efectivo_like("stg_utilities", "efectivo_backup", "UTILITY", util_cust, "UTILITIES", + "EFECTIVO_BACKUP", seen=cash_seen) fm3("efectivo_fm3", "EFECTIVO FM3") fm3("cheque_fm3", "CHEQUE FM3", check_col="num_cheque") billing("datos2", "datos2") @@ -217,6 +251,7 @@ def main(): print("=== Transactions load complete ===") print(f" skipped (unresolved customer): {skip_cust}") print(f" skipped (unparseable date) : {skip_date}") + print(f" skipped (EFECTIVO_BACKUP dup): {skip_dupe}") print(f" -> transactions : {count('transactions')}") print(f" by domain : {dict(by_dom)}") for src, n in by_src: