diff --git a/apps/api/src/customers/customers.service.ts b/apps/api/src/customers/customers.service.ts index 2e1c6b1..de9a1e1 100644 --- a/apps/api/src/customers/customers.service.ts +++ b/apps/api/src/customers/customers.service.ts @@ -45,6 +45,7 @@ export class CustomersService { select: { id: true, name: true, + nameSource: true, city: true, state: true, email: true, @@ -59,6 +60,7 @@ export class CustomersService { const items = rows.map((r) => ({ id: r.id, name: r.name, + nameSource: r.nameSource, city: r.city, state: r.state, email: r.email, diff --git a/apps/web/src/app/clientes/[id]/page.tsx b/apps/web/src/app/clientes/[id]/page.tsx index 170209a..2e752e6 100644 --- a/apps/web/src/app/clientes/[id]/page.tsx +++ b/apps/web/src/app/clientes/[id]/page.tsx @@ -10,6 +10,7 @@ import { formatMoney, serviceKindGlyph, serviceKindLabel, + SIN_NOMBRE, sourceSystemLabel, } from "@/lib/labels"; import type { @@ -139,7 +140,19 @@ function Hero({
-

{data.name}

+

+ {data.name} +

+ {data.nameSource && ( +
+ Nombre recuperado de {data.nameSource} — el registro original no + tenía nombre. +
+ )} {provenance && (
Origen: {provenance}
)} diff --git a/apps/web/src/app/clientes/page.tsx b/apps/web/src/app/clientes/page.tsx index caae065..f49c5bb 100644 --- a/apps/web/src/app/clientes/page.tsx +++ b/apps/web/src/app/clientes/page.tsx @@ -4,7 +4,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import Link from "next/link"; import { AppShell } from "@/components/AppShell"; import { getStats, listCustomers } from "@/lib/api"; -import { formatNumber } from "@/lib/labels"; +import { formatNumber, SIN_NOMBRE } from "@/lib/labels"; import type { BusinessLine, CustomerListItem, @@ -230,7 +230,17 @@ function CustomerRow({ c }: { c: CustomerListItem }) { {!c.status && ( )} - {c.name} + + {c.name} + + {c.nameSource && ( + + nombre recuperado + + )}
{location && {location}} diff --git a/apps/web/src/app/globals.css b/apps/web/src/app/globals.css index 433f77c..1bd73e8 100644 --- a/apps/web/src/app/globals.css +++ b/apps/web/src/app/globals.css @@ -799,6 +799,28 @@ button { background: var(--muted-2); flex: none; } +/* Legacy record with no name anywhere in the source — reads as absent data, + not as a customer actually called "(SIN NOMBRE)". */ +.cust-name-missing { + color: var(--muted-2); + font-style: italic; + font-weight: 400; +} +/* Name reconstructed from a secondary legacy table, so staff can tell it + apart from a name that was really on the customer record. */ +.name-source { + font-family: var(--font-sans); + font-size: 10px; + font-weight: 600; + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--muted-2); + border: 1px solid var(--line-strong); + border-radius: 4px; + padding: 2px 6px; + white-space: nowrap; + cursor: help; +} .cust-sub { display: flex; flex-wrap: wrap; @@ -930,6 +952,10 @@ button { letter-spacing: -0.015em; line-height: 1.1; } +.hero-name-missing { + font-style: italic; + color: rgba(251, 248, 240, 0.55); +} .hero-provenance { font-size: 12.5px; color: rgba(242, 239, 231, 0.62); diff --git a/apps/web/src/lib/labels.ts b/apps/web/src/lib/labels.ts index 40fa46c..cd4c0a7 100644 --- a/apps/web/src/lib/labels.ts +++ b/apps/web/src/lib/labels.ts @@ -2,6 +2,12 @@ import type { ServiceKind, TransactionDomain } from "./types"; +/** + * Placeholder the migration writes when a legacy record had no name and none + * could be recovered from a secondary table (migration/transform_customers.py). + */ +export const SIN_NOMBRE = "(SIN NOMBRE)"; + export const DOMAIN_LABELS: Record = { UTILITY: "Servicios", INSURANCE: "Seguros", diff --git a/apps/web/src/lib/types.ts b/apps/web/src/lib/types.ts index 4b7699d..473350b 100644 --- a/apps/web/src/lib/types.ts +++ b/apps/web/src/lib/types.ts @@ -23,7 +23,10 @@ export type BusinessLine = "utility" | "insurance" | "both"; export interface CustomerListItem { id: string; + /** May be "(SIN NOMBRE)" — the legacy record had no name and none was recoverable. */ name: string; + /** Legacy table the name was recovered from; null when it came from DATGRAL. */ + nameSource: string | null; city: string | null; state: string | null; email: string | null; @@ -193,6 +196,7 @@ export interface TransactionSummaryRow { export interface CustomerDetail { id: string; name: string; + nameSource: string | null; addressLine1: string | null; addressLine2: string | null; city: string | null; diff --git a/migration/run_all.py b/migration/run_all.py index be6964b..21868a3 100644 --- a/migration/run_all.py +++ b/migration/run_all.py @@ -17,6 +17,10 @@ Then: Reproducing dev -> prod is exactly `--env prod` (plus --stage if the staged Parquet isn't present on the machine running it). + +Note: the blob_extract step re-reads the original Access files directly (the +blobs are not in the staged Parquet), so the machine running this needs +SOURCE_ROOT + mdbtools + MinIO credentials even without --stage. """ from __future__ import annotations @@ -29,13 +33,18 @@ from pathlib import Path HERE = Path(__file__).parent PY = sys.executable # the venv python running this orchestrator -# Dependency order — extend as later modules land (policies, transactions, bank). +# Dependency order. Every step truncates what it owns, so anything downstream +# of a truncated table has to be rebuilt in the same pass — blob_extract is in +# this list because transform_properties and transform_policies truncate +# service_documents / policy_documents, which would otherwise leave the +# uploaded MinIO objects with no rows pointing at them. STEPS = [ "transform_customers.py", # customers + customer_legacy_refs (everything FKs to these) "transform_properties.py", # properties + services + trust accounts "transform_policies.py", # policies + installments/vehicles/drivers/benef/claims/adjusters "transform_transactions.py", # shared ledger + type_transactions + exchange_rates "transform_bank.py", # SCOTHIA bank register (no customer FK; independent) + "blob_extract.py", # document pointers; must follow properties + policies ] diff --git a/migration/transform_customers.py b/migration/transform_customers.py index 591a24e..1d49a8e 100644 --- a/migration/transform_customers.py +++ b/migration/transform_customers.py @@ -13,7 +13,13 @@ Rules come from the reconciliation pass (RECONCILIATION.md): `num_util` cross-reference. Matches fold into the existing customer (and enrich it with the ID-document fields the utilities master lacks); non-matches become new insurance-only customers. - - `COBRO3` is a charge batch, NOT a customer source -> excluded here. + - `COBRO3` is a charge batch, NOT a customer source -> excluded here + (it is still read as a name-recovery source, see below). + +`DATGRAL.NOMBRE` is blank on 266 legacy rows (140 utilities, 126 insurance). +Names for most of them are recovered from secondary tables — see +`_NAME_SOURCES` — and `customers.nameSource` records which table each +recovered name came from. Every legacy row folded in gets a `customer_legacy_refs` row (sourceSystem, sourceTable=DATGRAL, legacyId) so the merge is auditable and @@ -101,11 +107,69 @@ def as_decimal(v): return None +# -------------------------------- name recovery ----------------------------- # +NO_NAME = "(SIN NOMBRE)" + +# The old PHP importer skipped blank-name DATGRAL rows outright +# (jorgecuadros-intra-webapp/src/tools/customerAdapter.php:47,81). That also +# silently dropped those rows' properties, policies and transactions, because +# every other adapter resolved its customer FK through the customer_mapping +# table those skipped rows never got into (customerServiceAdapter.php:45), and +# customerBalanceAdapter.php:52 defaulted the unmapped ones to customer_id 0. +# Most blank-name rows are real accounts, so recover the name instead of +# skipping: 176 of the 257 carry a property, policy or transaction. +# +# Per side, most trustworthy source first; a later source only fills ids the +# earlier ones left unresolved. UTILSEG is the office's own hand-maintained +# name <-> id cross-reference spanning both lines; the rest are billing runs +# and policy rows that happen to repeat the customer's name. +# (label, staged source, table, id column, name column) +_NAME_SOURCES: dict[str, list[tuple[str, str, str, str, str]]] = { + "utilities": [ + ("UTILSEG", "stg_seguros", "utilseg", "util", "nombre"), + ("IVA 2015", "stg_utilities", "iva_2015", "num_id", "nombre"), + ("COBRO3", "stg_utilities", "cobro3", "num_id", "nombre"), + ], + "insurance": [ + ("UTILSEG", "stg_seguros", "utilseg", "seguros", "nombre"), + ("MULT", "stg_seguros", "mult", "num_id", "nombre_aseg"), + ("M EMPR", "stg_seguros", "m_empr", "num_id", "nombre_aseg"), + ("INCENDIO", "stg_seguros", "incendio", "num_id", "nombre_aseg"), + ], +} + + +def build_name_index(system: str) -> dict[str, tuple[str, str]]: + """legacy num_id -> (recovered name, source label) for one business line.""" + index: dict[str, tuple[str, str]] = {} + for label, source, table, id_col, name_col in _NAME_SOURCES[system]: + df = load(source, table) + if id_col not in df.columns or name_col not in df.columns: + raise KeyError(f"{table}: expected columns {id_col}/{name_col}, got {list(df.columns)}") + for _, row in df.iterrows(): + nid, name = norm_id(row[id_col]), s(row[name_col]) + if nid and name and nid not in index: + index[nid] = (name, label) + return index + + +def resolve_name(raw, nid, index) -> tuple[str, str | None]: + """(name, nameSource). nameSource stays None when DATGRAL had the name.""" + name = s(raw) + if name: + return name, None + if nid and nid in index: + return index[nid] + return NO_NAME, None + + # ------------------------------- record builders ---------------------------- # -def customer_from_utilities(row) -> dict: +def customer_from_utilities(row, name_index) -> dict: + name, name_source = resolve_name(row["nombre"], norm_id(row["num_id"]), name_index) return dict( id=str(uuid.uuid4()), - name=s(row["nombre"]) or "(SIN NOMBRE)", + name=name, + nameSource=name_source, addressLine1=s(row["direccion"]), addressLine2=s(row["colonia"]), city=s(row["ciudad"]), @@ -127,10 +191,12 @@ def customer_from_utilities(row) -> dict: ) -def customer_from_insurance(row) -> dict: +def customer_from_insurance(row, name_index) -> dict: + name, name_source = resolve_name(row["nombre"], norm_id(row["num_id"]), name_index) return dict( id=str(uuid.uuid4()), - name=s(row["nombre"]) or "(SIN NOMBRE)", + name=name, + nameSource=name_source, addressLine1=s(row["direccion_1"]), addressLine2=s(row["direccion_2"]), city=s(row["ciudad"]), @@ -153,7 +219,7 @@ def customer_from_insurance(row) -> dict: _CUST_COLS = [ - "id", "name", "addressLine1", "addressLine2", "city", "state", "zipCode", + "id", "name", "nameSource", "addressLine1", "addressLine2", "city", "state", "zipCode", "country", "phone", "mobile", "fax", "email", "notes", "identificationType", "identificationNumber", "identificationExpiration", "customerSince", "status", "feeAmount", "updatedAt", @@ -174,15 +240,19 @@ def main() -> None: util = load("stg_utilities", "datgral") ins = load("stg_seguros", "datgral") + util_names = build_name_index("utilities") + ins_names = build_name_index("insurance") customers: list[dict] = [] refs: list[tuple] = [] # (id, customerId, sourceSystem, sourceTable, legacyId) util_map: dict[str, str] = {} # utilities num_id -> customer_id + rec_by_id: dict[str, dict] = {} # Phase A: utilities DATGRAL = the master. for _, row in util.iterrows(): - rec = customer_from_utilities(row) + rec = customer_from_utilities(row, util_names) customers.append(rec) + rec_by_id[rec["id"]] = rec nid = norm_id(row["num_id"]) legacy = nid or f"rownum_{len(customers)}" refs.append((str(uuid.uuid4()), rec["id"], "utilities", "DATGRAL", legacy)) @@ -190,7 +260,7 @@ def main() -> None: util_map[nid] = rec["id"] # Phase B: insurance DATGRAL links via num_util, else new customer. - linked = new_ins = unmatched_numutil = 0 + linked = new_ins = unmatched_numutil = from_ins_side = 0 enrich: list[tuple] = [] # (customerId, insurance record) for fill-in for _, row in ins.iterrows(): ins_id = norm_id(row["num_id"]) or f"insrow_{new_ins+linked}" @@ -198,12 +268,22 @@ def main() -> None: if nutil and nutil in util_map: cust_id = util_map[nutil] linked += 1 - enrich.append((cust_id, customer_from_insurance(row))) + ins_rec = customer_from_insurance(row, ins_names) + # Last name-recovery path: a master whose own line had no name and + # no utilities-side fallback can still borrow the name its linked + # insurance record resolved to. + master = rec_by_id[cust_id] + if master["name"] == NO_NAME and ins_rec["name"] != NO_NAME: + master["name"] = ins_rec["name"] + master["nameSource"] = ins_rec["nameSource"] or "DATGRAL (seguros)" + from_ins_side += 1 + enrich.append((cust_id, ins_rec)) else: if nutil and nutil not in util_map: unmatched_numutil += 1 - rec = customer_from_insurance(row) + rec = customer_from_insurance(row, ins_names) customers.append(rec) + rec_by_id[rec["id"]] = rec cust_id = rec["id"] new_ins += 1 refs.append((str(uuid.uuid4()), cust_id, "insurance", "DATGRAL", ins_id)) @@ -251,6 +331,11 @@ def main() -> None: cur.execute("SELECT COUNT(*) FROM customer_legacy_refs " "GROUP BY customerId HAVING COUNT(*) > 1") merged = len(cur.fetchall()) + cur.execute("SELECT nameSource, COUNT(*) FROM customers " + "WHERE nameSource IS NOT NULL GROUP BY nameSource ORDER BY 2 DESC") + recovered = cur.fetchall() + cur.execute("SELECT COUNT(*) FROM customers WHERE name = %s", (NO_NAME,)) + still_unnamed = cur.fetchone()[0] print("=== Customer load complete ===") print(f" utilities DATGRAL rows : {len(util)}") @@ -262,6 +347,11 @@ def main() -> None: print(f" -> customer_legacy_refs : {n_refs} (expected {len(util)+len(ins)} = {len(util)+len(ins)})") print(f" refs by system : {by_sys}") print(f" customers with >1 ref (merged identities) : {merged}") + print(" name recovery (DATGRAL.NOMBRE was blank):") + for src, n in recovered: + print(f" from {src:16} : {n}") + print(f" of which via the linked insurance record : {from_ins_side}") + print(f" still {NO_NAME} : {still_unnamed}") assert n_cust == len(util) + new_ins, "customer count mismatch" assert n_refs == len(util) + len(ins), "legacy ref count mismatch" print(" validation: OK") diff --git a/packages/database/prisma/schema.prisma b/packages/database/prisma/schema.prisma index 504cf13..bc81dac 100644 --- a/packages/database/prisma/schema.prisma +++ b/packages/database/prisma/schema.prisma @@ -50,6 +50,11 @@ enum UserRole { model Customer { id String @id @default(uuid()) name String + // Which legacy table `name` actually came from. Null = DATGRAL.NOMBRE, the + // normal case. Anything else means DATGRAL's name was blank and the name was + // recovered from a secondary table (see migration/transform_customers.py), + // so staff can tell a reconstructed name from an original one. + nameSource String? addressLine1 String? addressLine2 String? city String?