Migration: recover blank customer names from secondary legacy tables

DATGRAL.NOMBRE is blank on 266 legacy rows (140 utilities, 126 insurance),
which surfaced in the UI as 257 customers literally named "(SIN NOMBRE)".
The blank is real — those cells are empty in the Access files, not lost in
extraction — but the rows mostly are not junk: 176 of the 257 carry a
property, a policy, or transactions.

The old PHP importer handled this by skipping blank-name rows outright
(jorgecuadros-intra-webapp/src/tools/customerAdapter.php:47,81). That was
worse than it looks: every other adapter resolved its customer FK through
the customer_mapping table those skipped rows never entered, so their
properties and policies were silently dropped (customerServiceAdapter.php:45)
and their transactions were written against customer_id 0
(customerBalanceAdapter.php:52). So: recover the name instead of skipping.

Names come from the secondary tables that still carry them, most trustworthy
first — UTILSEG (the office's own hand-maintained name <-> id cross-reference
spanning both lines), then the billing runs (IVA 2015, COBRO3) and the policy
rows' NOMBRE ASEG (MULT, M EMPR, INCENDIO). A linked customer can also borrow
the name its insurance record resolved to. Result: 213 of 257 recovered, 44
still genuinely nameless anywhere in the source.

customers.nameSource records which table each recovered name came from, so a
reconstructed name is never mistaken for one that was really on the record —
the list tags it "nombre recuperado", the detail header names the source, and
a still-unnamed customer renders muted italic instead of as a normal name.

Also fixes run_all.py: transform_properties and transform_policies truncate
service_documents/policy_documents, but blob_extract.py was not in the step
list, so a full re-run left the uploaded MinIO objects with no rows pointing
at them. Hit exactly that while reloading for this change.

Verified end-to-end: full pipeline re-run against dev reproduces every prior
count (1682 customers, 1519 properties, 2378 policies, 45861 transactions,
22354 bank rows, 70 documents) with zero orphans, and both apps build clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

apps/api/src/customers/customers.service.ts

@@ -45,6 +45,7 @@ export class CustomersService {
         select: {
           id: true,
           name: true,
+          nameSource: true,
           city: true,
           state: true,
           email: true,
@@ -59,6 +60,7 @@ export class CustomersService {
     const items = rows.map((r) => ({
       id: r.id,
       name: r.name,
+      nameSource: r.nameSource,
       city: r.city,
       state: r.state,
       email: r.email,

apps/web/src/app/clientes/[id]/page.tsx

@@ -10,6 +10,7 @@ import {
   formatMoney,
   serviceKindGlyph,
   serviceKindLabel,
+  SIN_NOMBRE,
   sourceSystemLabel,
 } from "@/lib/labels";
 import type {
@@ -139,7 +140,19 @@ function Hero({
     <div className="detail-hero">
       <div className="hero-top">
         <div>
-          <h1 className="hero-name">{data.name}</h1>
+          <h1
+            className={`hero-name${
+              data.name === SIN_NOMBRE ? " hero-name-missing" : ""
+            }`}
+          >
+            {data.name}
+          </h1>
+          {data.nameSource && (
+            <div className="hero-provenance">
+              Nombre recuperado de {data.nameSource} — el registro original no
+              tenía nombre.
+            </div>
+          )}
           {provenance && (
             <div className="hero-provenance">Origen: {provenance}</div>
           )}

apps/web/src/app/clientes/page.tsx

@@ -4,7 +4,7 @@ import { useCallback, useEffect, useRef, useState } from "react";
 import Link from "next/link";
 import { AppShell } from "@/components/AppShell";
 import { getStats, listCustomers } from "@/lib/api";
-import { formatNumber } from "@/lib/labels";
+import { formatNumber, SIN_NOMBRE } from "@/lib/labels";
 import type {
   BusinessLine,
   CustomerListItem,
@@ -230,7 +230,17 @@ function CustomerRow({ c }: { c: CustomerListItem }) {
           {!c.status && (
             <span className="inactive-dot" title="Inactivo" aria-hidden />
           )}
-          {c.name}
+          <span className={c.name === SIN_NOMBRE ? "cust-name-missing" : undefined}>
+            {c.name}
+          </span>
+          {c.nameSource && (
+            <span
+              className="name-source"
+              title={`El registro original no tenía nombre. Recuperado de ${c.nameSource}.`}
+            >
+              nombre recuperado
+            </span>
+          )}
         </div>
         <div className="cust-sub">
           {location && <span>{location}</span>}

apps/web/src/app/globals.css

@@ -799,6 +799,28 @@ button {
   background: var(--muted-2);
   flex: none;
 }
+/* Legacy record with no name anywhere in the source — reads as absent data,
+   not as a customer actually called "(SIN NOMBRE)". */
+.cust-name-missing {
+  color: var(--muted-2);
+  font-style: italic;
+  font-weight: 400;
+}
+/* Name reconstructed from a secondary legacy table, so staff can tell it
+   apart from a name that was really on the customer record. */
+.name-source {
+  font-family: var(--font-sans);
+  font-size: 10px;
+  font-weight: 600;
+  letter-spacing: 0.06em;
+  text-transform: uppercase;
+  color: var(--muted-2);
+  border: 1px solid var(--line-strong);
+  border-radius: 4px;
+  padding: 2px 6px;
+  white-space: nowrap;
+  cursor: help;
+}
 .cust-sub {
   display: flex;
   flex-wrap: wrap;
@@ -930,6 +952,10 @@ button {
   letter-spacing: -0.015em;
   line-height: 1.1;
 }
+.hero-name-missing {
+  font-style: italic;
+  color: rgba(251, 248, 240, 0.55);
+}
 .hero-provenance {
   font-size: 12.5px;
   color: rgba(242, 239, 231, 0.62);

apps/web/src/lib/labels.ts

@@ -2,6 +2,12 @@
 
 import type { ServiceKind, TransactionDomain } from "./types";
 
+/**
+ * Placeholder the migration writes when a legacy record had no name and none
+ * could be recovered from a secondary table (migration/transform_customers.py).
+ */
+export const SIN_NOMBRE = "(SIN NOMBRE)";
+
 export const DOMAIN_LABELS: Record<string, string> = {
   UTILITY: "Servicios",
   INSURANCE: "Seguros",

apps/web/src/lib/types.ts

@@ -23,7 +23,10 @@ export type BusinessLine = "utility" | "insurance" | "both";
 
 export interface CustomerListItem {
   id: string;
+  /** May be "(SIN NOMBRE)" — the legacy record had no name and none was recoverable. */
   name: string;
+  /** Legacy table the name was recovered from; null when it came from DATGRAL. */
+  nameSource: string | null;
   city: string | null;
   state: string | null;
   email: string | null;
@@ -193,6 +196,7 @@ export interface TransactionSummaryRow {
 export interface CustomerDetail {
   id: string;
   name: string;
+  nameSource: string | null;
   addressLine1: string | null;
   addressLine2: string | null;
   city: string | null;

migration/run_all.py

@@ -17,6 +17,10 @@ Then:
 
 Reproducing dev -> prod is exactly `--env prod` (plus --stage if the staged
 Parquet isn't present on the machine running it).
+
+Note: the blob_extract step re-reads the original Access files directly (the
+blobs are not in the staged Parquet), so the machine running this needs
+SOURCE_ROOT + mdbtools + MinIO credentials even without --stage.
 """
 
 from __future__ import annotations
@@ -29,13 +33,18 @@ from pathlib import Path
 HERE = Path(__file__).parent
 PY = sys.executable  # the venv python running this orchestrator
 
-# Dependency order — extend as later modules land (policies, transactions, bank).
+# Dependency order. Every step truncates what it owns, so anything downstream
+# of a truncated table has to be rebuilt in the same pass — blob_extract is in
+# this list because transform_properties and transform_policies truncate
+# service_documents / policy_documents, which would otherwise leave the
+# uploaded MinIO objects with no rows pointing at them.
 STEPS = [
     "transform_customers.py",     # customers + customer_legacy_refs (everything FKs to these)
     "transform_properties.py",    # properties + services + trust accounts
     "transform_policies.py",      # policies + installments/vehicles/drivers/benef/claims/adjusters
     "transform_transactions.py",  # shared ledger + type_transactions + exchange_rates
     "transform_bank.py",          # SCOTHIA bank register (no customer FK; independent)
+    "blob_extract.py",            # document pointers; must follow properties + policies
 ]
 
 

migration/transform_customers.py

@@ -13,7 +13,13 @@ Rules come from the reconciliation pass (RECONCILIATION.md):
     `num_util` cross-reference. Matches fold into the existing customer (and
     enrich it with the ID-document fields the utilities master lacks);
     non-matches become new insurance-only customers.
-  - `COBRO3` is a charge batch, NOT a customer source -> excluded here.
+  - `COBRO3` is a charge batch, NOT a customer source -> excluded here
+    (it is still read as a name-recovery source, see below).
+
+`DATGRAL.NOMBRE` is blank on 266 legacy rows (140 utilities, 126 insurance).
+Names for most of them are recovered from secondary tables — see
+`_NAME_SOURCES` — and `customers.nameSource` records which table each
+recovered name came from.
 
 Every legacy row folded in gets a `customer_legacy_refs` row
 (sourceSystem, sourceTable=DATGRAL, legacyId) so the merge is auditable and
@@ -101,11 +107,69 @@ def as_decimal(v):
         return None
 
 
+# -------------------------------- name recovery ----------------------------- #
+NO_NAME = "(SIN NOMBRE)"
+
+# The old PHP importer skipped blank-name DATGRAL rows outright
+# (jorgecuadros-intra-webapp/src/tools/customerAdapter.php:47,81). That also
+# silently dropped those rows' properties, policies and transactions, because
+# every other adapter resolved its customer FK through the customer_mapping
+# table those skipped rows never got into (customerServiceAdapter.php:45), and
+# customerBalanceAdapter.php:52 defaulted the unmapped ones to customer_id 0.
+# Most blank-name rows are real accounts, so recover the name instead of
+# skipping: 176 of the 257 carry a property, policy or transaction.
+#
+# Per side, most trustworthy source first; a later source only fills ids the
+# earlier ones left unresolved. UTILSEG is the office's own hand-maintained
+# name <-> id cross-reference spanning both lines; the rest are billing runs
+# and policy rows that happen to repeat the customer's name.
+#     (label, staged source, table, id column, name column)
+_NAME_SOURCES: dict[str, list[tuple[str, str, str, str, str]]] = {
+    "utilities": [
+        ("UTILSEG",  "stg_seguros",   "utilseg",  "util",    "nombre"),
+        ("IVA 2015", "stg_utilities", "iva_2015", "num_id",  "nombre"),
+        ("COBRO3",   "stg_utilities", "cobro3",   "num_id",  "nombre"),
+    ],
+    "insurance": [
+        ("UTILSEG",  "stg_seguros",   "utilseg",  "seguros", "nombre"),
+        ("MULT",     "stg_seguros",   "mult",     "num_id",  "nombre_aseg"),
+        ("M EMPR",   "stg_seguros",   "m_empr",   "num_id",  "nombre_aseg"),
+        ("INCENDIO", "stg_seguros",   "incendio", "num_id",  "nombre_aseg"),
+    ],
+}
+
+
+def build_name_index(system: str) -> dict[str, tuple[str, str]]:
+    """legacy num_id -> (recovered name, source label) for one business line."""
+    index: dict[str, tuple[str, str]] = {}
+    for label, source, table, id_col, name_col in _NAME_SOURCES[system]:
+        df = load(source, table)
+        if id_col not in df.columns or name_col not in df.columns:
+            raise KeyError(f"{table}: expected columns {id_col}/{name_col}, got {list(df.columns)}")
+        for _, row in df.iterrows():
+            nid, name = norm_id(row[id_col]), s(row[name_col])
+            if nid and name and nid not in index:
+                index[nid] = (name, label)
+    return index
+
+
+def resolve_name(raw, nid, index) -> tuple[str, str | None]:
+    """(name, nameSource). nameSource stays None when DATGRAL had the name."""
+    name = s(raw)
+    if name:
+        return name, None
+    if nid and nid in index:
+        return index[nid]
+    return NO_NAME, None
+
+
 # ------------------------------- record builders ---------------------------- #
-def customer_from_utilities(row) -> dict:
+def customer_from_utilities(row, name_index) -> dict:
+    name, name_source = resolve_name(row["nombre"], norm_id(row["num_id"]), name_index)
     return dict(
         id=str(uuid.uuid4()),
-        name=s(row["nombre"]) or "(SIN NOMBRE)",
+        name=name,
+        nameSource=name_source,
         addressLine1=s(row["direccion"]),
         addressLine2=s(row["colonia"]),
         city=s(row["ciudad"]),
@@ -127,10 +191,12 @@ def customer_from_utilities(row) -> dict:
     )
 
 
-def customer_from_insurance(row) -> dict:
+def customer_from_insurance(row, name_index) -> dict:
+    name, name_source = resolve_name(row["nombre"], norm_id(row["num_id"]), name_index)
     return dict(
         id=str(uuid.uuid4()),
-        name=s(row["nombre"]) or "(SIN NOMBRE)",
+        name=name,
+        nameSource=name_source,
         addressLine1=s(row["direccion_1"]),
         addressLine2=s(row["direccion_2"]),
         city=s(row["ciudad"]),
@@ -153,7 +219,7 @@ def customer_from_insurance(row) -> dict:
 
 
 _CUST_COLS = [
-    "id", "name", "addressLine1", "addressLine2", "city", "state", "zipCode",
+    "id", "name", "nameSource", "addressLine1", "addressLine2", "city", "state", "zipCode",
     "country", "phone", "mobile", "fax", "email", "notes", "identificationType",
     "identificationNumber", "identificationExpiration", "customerSince",
     "status", "feeAmount", "updatedAt",
@@ -174,15 +240,19 @@ def main() -> None:
 
     util = load("stg_utilities", "datgral")
     ins = load("stg_seguros", "datgral")
+    util_names = build_name_index("utilities")
+    ins_names = build_name_index("insurance")
 
     customers: list[dict] = []
     refs: list[tuple] = []          # (id, customerId, sourceSystem, sourceTable, legacyId)
     util_map: dict[str, str] = {}   # utilities num_id -> customer_id
+    rec_by_id: dict[str, dict] = {}
 
     # Phase A: utilities DATGRAL = the master.
     for _, row in util.iterrows():
-        rec = customer_from_utilities(row)
+        rec = customer_from_utilities(row, util_names)
         customers.append(rec)
+        rec_by_id[rec["id"]] = rec
         nid = norm_id(row["num_id"])
         legacy = nid or f"rownum_{len(customers)}"
         refs.append((str(uuid.uuid4()), rec["id"], "utilities", "DATGRAL", legacy))
@@ -190,7 +260,7 @@ def main() -> None:
             util_map[nid] = rec["id"]
 
     # Phase B: insurance DATGRAL links via num_util, else new customer.
-    linked = new_ins = unmatched_numutil = 0
+    linked = new_ins = unmatched_numutil = from_ins_side = 0
     enrich: list[tuple] = []  # (customerId, insurance record) for fill-in
     for _, row in ins.iterrows():
         ins_id = norm_id(row["num_id"]) or f"insrow_{new_ins+linked}"
@@ -198,12 +268,22 @@ def main() -> None:
         if nutil and nutil in util_map:
             cust_id = util_map[nutil]
             linked += 1
-            enrich.append((cust_id, customer_from_insurance(row)))
+            ins_rec = customer_from_insurance(row, ins_names)
+            # Last name-recovery path: a master whose own line had no name and
+            # no utilities-side fallback can still borrow the name its linked
+            # insurance record resolved to.
+            master = rec_by_id[cust_id]
+            if master["name"] == NO_NAME and ins_rec["name"] != NO_NAME:
+                master["name"] = ins_rec["name"]
+                master["nameSource"] = ins_rec["nameSource"] or "DATGRAL (seguros)"
+                from_ins_side += 1
+            enrich.append((cust_id, ins_rec))
         else:
             if nutil and nutil not in util_map:
                 unmatched_numutil += 1
-            rec = customer_from_insurance(row)
+            rec = customer_from_insurance(row, ins_names)
             customers.append(rec)
+            rec_by_id[rec["id"]] = rec
             cust_id = rec["id"]
             new_ins += 1
         refs.append((str(uuid.uuid4()), cust_id, "insurance", "DATGRAL", ins_id))
@@ -251,6 +331,11 @@ def main() -> None:
     cur.execute("SELECT COUNT(*) FROM customer_legacy_refs "
                 "GROUP BY customerId HAVING COUNT(*) > 1")
     merged = len(cur.fetchall())
+    cur.execute("SELECT nameSource, COUNT(*) FROM customers "
+                "WHERE nameSource IS NOT NULL GROUP BY nameSource ORDER BY 2 DESC")
+    recovered = cur.fetchall()
+    cur.execute("SELECT COUNT(*) FROM customers WHERE name = %s", (NO_NAME,))
+    still_unnamed = cur.fetchone()[0]
 
     print("=== Customer load complete ===")
     print(f"  utilities DATGRAL rows : {len(util)}")
@@ -262,6 +347,11 @@ def main() -> None:
     print(f"  -> customer_legacy_refs : {n_refs}  (expected {len(util)+len(ins)} = {len(util)+len(ins)})")
     print(f"     refs by system       : {by_sys}")
     print(f"     customers with >1 ref (merged identities) : {merged}")
+    print("  name recovery (DATGRAL.NOMBRE was blank):")
+    for src, n in recovered:
+        print(f"     from {src:16} : {n}")
+    print(f"     of which via the linked insurance record : {from_ins_side}")
+    print(f"     still {NO_NAME} : {still_unnamed}")
     assert n_cust == len(util) + new_ins, "customer count mismatch"
     assert n_refs == len(util) + len(ins), "legacy ref count mismatch"
     print("  validation: OK")

packages/database/prisma/schema.prisma

@@ -50,6 +50,11 @@ enum UserRole {
 model Customer {
   id                        String    @id @default(uuid())
   name                      String
+  // Which legacy table `name` actually came from. Null = DATGRAL.NOMBRE, the
+  // normal case. Anything else means DATGRAL's name was blank and the name was
+  // recovered from a secondary table (see migration/transform_customers.py),
+  // so staff can tell a reconstructed name from an original one.
+  nameSource                String?
   addressLine1              String?
   addressLine2              String?
   city                      String?