Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8c144fe8c4 | ||
|
|
4f2f064955 | ||
|
|
2f99bd5f98 | ||
|
|
458b2b272d | ||
|
|
81938877ed | ||
|
|
d854dff091 | ||
|
|
19864f16f2 |
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@jorgecuadros/api",
|
"name": "@jorgecuadros/api",
|
||||||
"version": "1.0.19",
|
"version": "1.0.22",
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "nest build",
|
"build": "nest build",
|
||||||
|
|||||||
@@ -0,0 +1,161 @@
|
|||||||
|
import {
|
||||||
|
nameTokens,
|
||||||
|
suggestCustomersByName,
|
||||||
|
suggestionNote,
|
||||||
|
type CustomerNameRow,
|
||||||
|
} from "./name-matcher";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Every row here is a real name out of the customer book (1536 rows, dev
|
||||||
|
* mirror of production), chosen because it is one of the shapes that breaks
|
||||||
|
* naive matching: surname-first ordering, a middle initial, a Spanish double
|
||||||
|
* surname, a joint account, a missing comma, and the `(SIN NOMBRE)`
|
||||||
|
* placeholder the migration left for customers whose DATGRAL row had no name.
|
||||||
|
*/
|
||||||
|
const BOOK: CustomerNameRow[] = [
|
||||||
|
{ id: "c1", name: "WAGONER, PAMELA" },
|
||||||
|
{ id: "c2", name: "MCWILLIAMS, BRIAN MICHAEL" },
|
||||||
|
{ id: "c3", name: "MCWILLIAMS, BRIAN" },
|
||||||
|
{ id: "c4", name: "WEAKLAND, RICHARD E." },
|
||||||
|
{ id: "c5", name: "ESTRADA, JERRY & MARILYN" },
|
||||||
|
{ id: "c6", name: "CABALLERO PRIETO, GUILLERMO" },
|
||||||
|
{ id: "c7", name: "GREENE STEPHANIE" },
|
||||||
|
{ id: "c8", name: "(SIN NOMBRE)" },
|
||||||
|
{ id: "c9", name: "MUÑOZ, LUIS ALBERTO" },
|
||||||
|
{ id: "c10", name: "SMITH, DANIEL" },
|
||||||
|
{ id: "c11", name: "SMITH, JOHN" },
|
||||||
|
];
|
||||||
|
|
||||||
|
describe("nameTokens", () => {
|
||||||
|
it("makes the two orderings the same set", () => {
|
||||||
|
expect(nameTokens("PAMELA WAGONER").sort()).toEqual(
|
||||||
|
nameTokens("WAGONER, PAMELA").sort(),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops initials, particles and corporate suffixes", () => {
|
||||||
|
expect(nameTokens("WEAKLAND, RICHARD E.")).toEqual(["WEAKLAND", "RICHARD"]);
|
||||||
|
expect(nameTokens("GARCIA DE LA TORRE, ANA")).toEqual(["GARCIA", "TORRE", "ANA"]);
|
||||||
|
expect(nameTokens("CONSTRUCTORA BAJA S.A. DE C.V.")).toEqual([
|
||||||
|
"CONSTRUCTORA",
|
||||||
|
"BAJA",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("folds accents so OCR's MUNOZ reaches the book's MUÑOZ", () => {
|
||||||
|
expect(nameTokens("MUÑOZ")).toEqual(["MUNOZ"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("drops the phone number ANA prints against the insured name", () => {
|
||||||
|
// Observed verbatim from the ANA automobile face.
|
||||||
|
expect(nameTokens("MARIA GARCIA Ph.3102001538")).toEqual([
|
||||||
|
"MARIA",
|
||||||
|
"GARCIA",
|
||||||
|
"PH",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("suggestCustomersByName", () => {
|
||||||
|
it("matches the reversed name exactly", () => {
|
||||||
|
const [top] = suggestCustomersByName("PAMELA WAGONER", BOOK);
|
||||||
|
expect(top).toMatchObject({ customerId: "c1", tier: "EXACT", score: 1 });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("treats a printed middle name the book lacks as a partial hit", () => {
|
||||||
|
const hits = suggestCustomersByName("PAMELA DENISE WAGONER", BOOK);
|
||||||
|
expect(hits[0]).toMatchObject({ customerId: "c1", tier: "PARTIAL" });
|
||||||
|
expect(hits[0].score).toBeCloseTo(2 / 3);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ranks the exact row above the row that merely contains it", () => {
|
||||||
|
// Both MCWILLIAMS rows are reachable from this name; the one that holds
|
||||||
|
// the middle name is the exact set and must come first.
|
||||||
|
const hits = suggestCustomersByName("BRIAN MICHAEL MCWILLIAMS", BOOK);
|
||||||
|
expect(hits.map((h) => h.customerId)).toEqual(["c2", "c3"]);
|
||||||
|
expect(hits[0].tier).toBe("EXACT");
|
||||||
|
expect(hits[1].tier).toBe("PARTIAL");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reaches a joint account from the one spouse the carrier printed", () => {
|
||||||
|
const hits = suggestCustomersByName("JERRY ESTRADA", BOOK);
|
||||||
|
expect(hits[0]).toMatchObject({ customerId: "c5", tier: "PARTIAL" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("will not reach a joint account on given names alone", () => {
|
||||||
|
// No surname printed: `JERRY MARILYN` overlaps ESTRADA, JERRY & MARILYN
|
||||||
|
// on two tokens, and matching on that would book a stranger's policy.
|
||||||
|
expect(suggestCustomersByName("JERRY MARILYN", BOOK)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a Spanish double surname regardless of where the comma fell", () => {
|
||||||
|
const [top] = suggestCustomersByName("GUILLERMO CABALLERO PRIETO", BOOK);
|
||||||
|
expect(top).toMatchObject({ customerId: "c6", tier: "EXACT" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("still matches a book row that has no comma", () => {
|
||||||
|
const [top] = suggestCustomersByName("STEPHANIE GREENE", BOOK);
|
||||||
|
expect(top).toMatchObject({ customerId: "c7", tier: "EXACT" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("never suggests the (SIN NOMBRE) placeholder", () => {
|
||||||
|
expect(suggestCustomersByName("SIN NOMBRE", BOOK)).toEqual([]);
|
||||||
|
expect(suggestCustomersByName("NOMBRE DEL ASEGURADO", BOOK)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns nothing on a shared surname alone", () => {
|
||||||
|
// 185 surnames are shared by 524 customers; one token is not evidence.
|
||||||
|
expect(suggestCustomersByName("SMITH", BOOK)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns nothing for a different person with the same surname", () => {
|
||||||
|
expect(suggestCustomersByName("ROBERT SMITH", BOOK)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses a page-sized blob", () => {
|
||||||
|
// GMX's especificación has no field labels and the parser has handed its
|
||||||
|
// whole first page over as the insured name.
|
||||||
|
const blob =
|
||||||
|
"ESPECIFICACION DE LA POLIZA DE SEGURO DE RESPONSABILIDAD CIVIL " +
|
||||||
|
"EXPEDIDA A FAVOR DE PAMELA WAGONER CON VIGENCIA DEL 01 DE ENERO";
|
||||||
|
expect(suggestCustomersByName(blob, BOOK)).toEqual([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("caps the list", () => {
|
||||||
|
expect(suggestCustomersByName("BRIAN MICHAEL MCWILLIAMS", BOOK, 1)).toHaveLength(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("handles a null insured name", () => {
|
||||||
|
expect(suggestCustomersByName(null, BOOK)).toEqual([]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("suggestionNote", () => {
|
||||||
|
it("says nothing when there is nothing", () => {
|
||||||
|
expect(suggestionNote([])).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("names a single exact hit", () => {
|
||||||
|
expect(suggestionNote(suggestCustomersByName("PAMELA WAGONER", BOOK))).toBe(
|
||||||
|
"posible cliente por nombre: WAGONER, PAMELA",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a tie rather than picking one", () => {
|
||||||
|
// The book really does hold EMERY, LAURA twice and KIRCHHOFF, CINDY
|
||||||
|
// three times.
|
||||||
|
const dupes: CustomerNameRow[] = [
|
||||||
|
{ id: "d1", name: "EMERY, LAURA" },
|
||||||
|
{ id: "d2", name: "EMERY, LAURA" },
|
||||||
|
];
|
||||||
|
expect(suggestionNote(suggestCustomersByName("LAURA EMERY", dupes))).toBe(
|
||||||
|
"2 clientes tienen ese mismo nombre; elija cuál",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("lists partial hits", () => {
|
||||||
|
expect(suggestionNote(suggestCustomersByName("PAMELA DENISE WAGONER", BOOK))).toBe(
|
||||||
|
"posibles clientes por nombre: WAGONER, PAMELA",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
/**
|
||||||
|
* Suggests which existing customer a printed insured name belongs to.
|
||||||
|
*
|
||||||
|
* The office books customers surname-first ("WAGONER, PAMELA") and carriers
|
||||||
|
* print them given-name-first ("PAMELA DENISE WAGONER"), so a string compare
|
||||||
|
* never hits. Comparing *token sets* does, and it is order-insensitive by
|
||||||
|
* construction — which is the whole trick.
|
||||||
|
*
|
||||||
|
* **These are suggestions, never matches.** Nothing here sets
|
||||||
|
* `matchedCustomerId` or `confident`; the review screen offers the ranked
|
||||||
|
* names and a human picks. That line is not caution, it is what the book
|
||||||
|
* measures out to: of 1536 customers, 1487 have a distinct normalized token
|
||||||
|
* set — but loosen the rule to surname + first given name only and 131 of
|
||||||
|
* them (8.5%) collide, because the book holds `MCWILLIAMS, BRIAN MICHAEL`
|
||||||
|
* *and* `MCWILLIAMS, BRIAN`, and `CUADROS, JORGE JR` alongside three
|
||||||
|
* `CUADROS, JORGE H.`. 185 surnames are shared by 524 customers, so a
|
||||||
|
* surname alone carries no information at all.
|
||||||
|
*
|
||||||
|
* The two tiers below are drawn at the two places that measurement puts a
|
||||||
|
* cliff: full token-set equality, where cross-person collisions are
|
||||||
|
* effectively zero, and strict containment, where they are common enough
|
||||||
|
* that the result can only ever be a hint.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** A customer row as the matcher needs it — id and the book's name. */
|
||||||
|
export interface CustomerNameRow {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type NameMatchTier = "EXACT" | "PARTIAL";
|
||||||
|
|
||||||
|
export interface CustomerNameSuggestion {
|
||||||
|
customerId: string;
|
||||||
|
customerName: string;
|
||||||
|
/**
|
||||||
|
* `EXACT` — the two names carry the same tokens, in any order.
|
||||||
|
* `PARTIAL` — one name's tokens are all present in the other's, plus the
|
||||||
|
* surname. A printed middle name the book does not hold, or a joint
|
||||||
|
* account where the carrier named one spouse, both land here.
|
||||||
|
*/
|
||||||
|
tier: NameMatchTier;
|
||||||
|
/** Shared tokens over the longer name's token count, 0..1. */
|
||||||
|
score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Words that carry no identity. Spanish particles and the ampersand joining
|
||||||
|
* a couple are noise; the corporate suffixes are dropped so `S.A. DE C.V.`
|
||||||
|
* does not make every company look alike.
|
||||||
|
*/
|
||||||
|
const NOISE = new Set([
|
||||||
|
"DE", "DEL", "LA", "LAS", "LOS", "Y", "AND", "VDA",
|
||||||
|
"JR", "SR", "II", "III", "IV",
|
||||||
|
"SA", "CV", "SAPI", "SRL", "RL", "SC", "INC", "LLC", "LTD", "CORP", "CO",
|
||||||
|
]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Placeholder rows the migration left behind. Fourteen customers are named
|
||||||
|
* literally `(SIN NOMBRE)`; without this they would be one 14-way tie on
|
||||||
|
* every unreadable name.
|
||||||
|
*/
|
||||||
|
const PLACEHOLDER = new Set(["SIN NOMBRE", "NOMBRE SIN"]);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A name blob longer than this is not a name. GMX's PVL especificación has
|
||||||
|
* no field labels, and the parser has been seen handing its entire first
|
||||||
|
* page over as `insuredName`; matching that against the book would find
|
||||||
|
* a surname somewhere in the prose and suggest a stranger.
|
||||||
|
*/
|
||||||
|
const MAX_TOKENS = 8;
|
||||||
|
const MAX_CHARS = 80;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Splits a name into comparable tokens.
|
||||||
|
*
|
||||||
|
* Accents go first, and deliberately in both directions: the book holds
|
||||||
|
* `MUÑOZ` where OCR routinely reads `MUNOZ`, and folding both to the same
|
||||||
|
* ASCII makes that a hit rather than a miss.
|
||||||
|
*
|
||||||
|
* Tokens containing digits are dropped outright. ANA's automobile face
|
||||||
|
* prints the phone number hard against the insured name — the parser has
|
||||||
|
* emitted `MARIA GARCIA Ph.3102001538` — and the digits would otherwise
|
||||||
|
* be an extra token forever blocking `EXACT`.
|
||||||
|
*
|
||||||
|
* Single letters are dropped as initials: the book is full of
|
||||||
|
* `WEAKLAND, RICHARD E.`, and a carrier that prints the middle name in
|
||||||
|
* full should still match the row that abbreviates it.
|
||||||
|
*/
|
||||||
|
export function nameTokens(raw: string): string[] {
|
||||||
|
const cleaned = raw
|
||||||
|
.normalize("NFD")
|
||||||
|
.replace(/[\u0300-\u036f]/g, "")
|
||||||
|
.toUpperCase()
|
||||||
|
.replace(/[^A-Z0-9]+/g, " ")
|
||||||
|
.trim();
|
||||||
|
|
||||||
|
const tokens = cleaned
|
||||||
|
.split(" ")
|
||||||
|
.filter((t) => t.length > 1 && !/\d/.test(t) && !NOISE.has(t));
|
||||||
|
|
||||||
|
return [...new Set(tokens)];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The surname tokens — everything before the comma the book writes. */
|
||||||
|
function surnameTokens(bookName: string): string[] {
|
||||||
|
const comma = bookName.indexOf(",");
|
||||||
|
// 54 of 1536 rows have no comma at all ("GREENE STEPHANIE",
|
||||||
|
// "FAROOQ VAKIL"), and which half is the surname is unknowable. Requiring
|
||||||
|
// a surname we cannot identify would silently exclude those rows, so they
|
||||||
|
// fall back to requiring nothing beyond the containment rule.
|
||||||
|
if (comma < 0) return [];
|
||||||
|
return nameTokens(bookName.slice(0, comma));
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPlaceholder(tokens: string[]): boolean {
|
||||||
|
return tokens.length === 0 || PLACEHOLDER.has([...tokens].sort().join(" "));
|
||||||
|
}
|
||||||
|
|
||||||
|
function containsAll(haystack: Set<string>, needles: string[]): boolean {
|
||||||
|
return needles.every((n) => haystack.has(n));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ranks the book against one printed name.
|
||||||
|
*
|
||||||
|
* Returns at most `limit` suggestions, `EXACT` before `PARTIAL` and higher
|
||||||
|
* score first. An empty array means the printed name was unusable (too
|
||||||
|
* long, too few real tokens) or nothing in the book came close — both of
|
||||||
|
* which leave the review screen exactly as it is today.
|
||||||
|
*/
|
||||||
|
export function suggestCustomersByName(
|
||||||
|
printedName: string | null | undefined,
|
||||||
|
customers: CustomerNameRow[],
|
||||||
|
limit = 3,
|
||||||
|
): CustomerNameSuggestion[] {
|
||||||
|
if (!printedName || printedName.length > MAX_CHARS) return [];
|
||||||
|
|
||||||
|
const printed = nameTokens(printedName);
|
||||||
|
// One usable token is a surname or a given name on its own, and 34% of the
|
||||||
|
// book shares a surname with someone. Nothing useful can come of it.
|
||||||
|
if (printed.length < 2 || printed.length > MAX_TOKENS) return [];
|
||||||
|
|
||||||
|
const printedSet = new Set(printed);
|
||||||
|
const out: CustomerNameSuggestion[] = [];
|
||||||
|
|
||||||
|
for (const c of customers) {
|
||||||
|
const book = nameTokens(c.name);
|
||||||
|
if (isPlaceholder(book) || book.length < 2) continue;
|
||||||
|
|
||||||
|
const bookSet = new Set(book);
|
||||||
|
const overlap = printed.filter((t) => bookSet.has(t)).length;
|
||||||
|
// Two shared tokens is the floor: one is a bare surname collision.
|
||||||
|
if (overlap < 2) continue;
|
||||||
|
|
||||||
|
const bookInPrinted = containsAll(printedSet, book);
|
||||||
|
const printedInBook = containsAll(bookSet, printed);
|
||||||
|
if (!bookInPrinted && !printedInBook) continue;
|
||||||
|
|
||||||
|
// When the book's name is the shorter one, containment already proves
|
||||||
|
// the surname was printed. When the printed name is shorter — the book
|
||||||
|
// holds a middle name or a second spouse the carrier omitted — the
|
||||||
|
// surname must be there explicitly, or `JERRY MARILYN` would match
|
||||||
|
// `ESTRADA, JERRY & MARILYN` on given names alone.
|
||||||
|
if (!bookInPrinted && !containsAll(printedSet, surnameTokens(c.name))) continue;
|
||||||
|
|
||||||
|
out.push({
|
||||||
|
customerId: c.id,
|
||||||
|
customerName: c.name,
|
||||||
|
tier: bookInPrinted && printedInBook ? "EXACT" : "PARTIAL",
|
||||||
|
score: overlap / Math.max(book.length, printed.length),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
out.sort((a, b) => {
|
||||||
|
if (a.tier !== b.tier) return a.tier === "EXACT" ? -1 : 1;
|
||||||
|
if (b.score !== a.score) return b.score - a.score;
|
||||||
|
return a.customerName.localeCompare(b.customerName);
|
||||||
|
});
|
||||||
|
|
||||||
|
return out.slice(0, limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Review-queue wording for what the suggestions amount to. */
|
||||||
|
export function suggestionNote(suggestions: CustomerNameSuggestion[]): string | null {
|
||||||
|
if (suggestions.length === 0) return null;
|
||||||
|
|
||||||
|
const exact = suggestions.filter((s) => s.tier === "EXACT");
|
||||||
|
// More than one exact hit is the duplicate-customer case the book really
|
||||||
|
// has (`EMERY, LAURA` twice, `KIRCHHOFF, CINDY` three times). Saying so is
|
||||||
|
// more useful than naming whichever one sorted first.
|
||||||
|
if (exact.length > 1) {
|
||||||
|
return `${exact.length} clientes tienen ese mismo nombre; elija cuál`;
|
||||||
|
}
|
||||||
|
if (exact.length === 1) {
|
||||||
|
return `posible cliente por nombre: ${exact[0].customerName}`;
|
||||||
|
}
|
||||||
|
return `posibles clientes por nombre: ${suggestions.map((s) => s.customerName).join(", ")}`;
|
||||||
|
}
|
||||||
@@ -859,6 +859,15 @@ describe("parsePolicy / ANA driver's policy (licencia)", () => {
|
|||||||
expect(parsePolicy(tripled).drivers).toHaveLength(1);
|
expect(parsePolicy(tripled).drivers).toHaveLength(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("splits the phone off the name even without the printed column gap", () => {
|
||||||
|
// The phone shares the name cell, and the only thing marking it off is
|
||||||
|
// white space — which the OCR seam is free to collapse. Depending on the
|
||||||
|
// gap surviving is what put "PAMELA DENISE WAGONER Ph.3102001538" in the
|
||||||
|
// insured field, where it matched no customer.
|
||||||
|
const collapsed = page(ANA_LICENCIA.text.replace(/ {2,}/g, " "));
|
||||||
|
expect(parsePolicy(collapsed).insuredName).toBe("PAMELA DENISE WAGONER");
|
||||||
|
});
|
||||||
|
|
||||||
it("drops the four empty driver slots", () => {
|
it("drops the four empty driver slots", () => {
|
||||||
// Slots 2-5 print an empty NAME and a bare "NONE" licence.
|
// Slots 2-5 print an empty NAME and a bare "NONE" licence.
|
||||||
expect(p.drivers.map((d) => d.fullName)).toEqual(["PAMELA DENISE WAGONER"]);
|
expect(p.drivers.map((d) => d.fullName)).toEqual(["PAMELA DENISE WAGONER"]);
|
||||||
|
|||||||
@@ -727,13 +727,22 @@ function parseGmxEspecificacion(page: OcrPage): ParsedPolicy {
|
|||||||
* value in a right-hand column, so a two-line risk location comes back as the
|
* value in a right-hand column, so a two-line risk location comes back as the
|
||||||
* label line plus a continuation line indented to the same column. A blank
|
* label line plus a continuation line indented to the same column. A blank
|
||||||
* line always terminates the cell.
|
* line always terminates the cell.
|
||||||
|
*
|
||||||
|
* The line budget is a guard, not the layout: the longest cell on this
|
||||||
|
* document wraps once. It exists because "walk until the cell ends" is only
|
||||||
|
* as good as the blank line it walks to, and when the OCR seam stopped
|
||||||
|
* emitting those, this returned the whole first page as the insured's name —
|
||||||
|
* a failure with no bad value to notice, just one enormous good one.
|
||||||
*/
|
*/
|
||||||
|
const ESPEC_MAX_WRAP = 3;
|
||||||
|
|
||||||
function espectBlock(lines: string[], label: RegExp): string | null {
|
function espectBlock(lines: string[], label: RegExp): string | null {
|
||||||
for (let i = 0; i < lines.length; i++) {
|
for (let i = 0; i < lines.length; i++) {
|
||||||
const m = lines[i].match(label);
|
const m = lines[i].match(label);
|
||||||
if (!m?.[1]) continue;
|
if (!m?.[1]) continue;
|
||||||
const parts = [m[1]];
|
const parts = [m[1]];
|
||||||
for (let j = i + 1; j < lines.length && lines[j].trim(); j++) {
|
const until = Math.min(lines.length, i + 1 + ESPEC_MAX_WRAP);
|
||||||
|
for (let j = i + 1; j < until && lines[j].trim(); j++) {
|
||||||
parts.push(lines[j].trim());
|
parts.push(lines[j].trim());
|
||||||
}
|
}
|
||||||
const value = parts.join(" ").replace(/\s+/g, " ").trim();
|
const value = parts.join(" ").replace(/\s+/g, " ").trim();
|
||||||
@@ -1889,7 +1898,12 @@ function parseAnaDrivers(lines: string[]): ParsedDriver[] {
|
|||||||
const raw = scope[at].match(/^\s*\d\.\s*NAME\s*:\s*(.*)$/i)?.[1]?.trim() ?? "";
|
const raw = scope[at].match(/^\s*\d\.\s*NAME\s*:\s*(.*)$/i)?.[1]?.trim() ?? "";
|
||||||
if (!raw) return;
|
if (!raw) return;
|
||||||
|
|
||||||
const phoneAt = raw.match(/\s{2,}Ph\.?\s*([\d()\s.-]{7,})\s*$/i);
|
// One space is enough to split on: the "Ph." marker plus seven digits at
|
||||||
|
// the end of the cell is not something a name does. Requiring the printed
|
||||||
|
// column gap made this depend on the reassembler keeping it, and when that
|
||||||
|
// collapsed the phone rode into `insuredName` ("PAMELA DENISE WAGONER
|
||||||
|
// Ph.3102001538") and no customer matched it.
|
||||||
|
const phoneAt = raw.match(/\s+Ph\.?\s*([\d()\s.-]{7,})\s*$/i);
|
||||||
const fullName = (phoneAt ? raw.slice(0, phoneAt.index) : raw).replace(/\s+/g, " ").trim();
|
const fullName = (phoneAt ? raw.slice(0, phoneAt.index) : raw).replace(/\s+/g, " ").trim();
|
||||||
if (!fullName) return;
|
if (!fullName) return;
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import { PolicyMatcherService } from "./policy-matcher.service";
|
||||||
|
import type { PrismaService } from "../prisma/prisma.service";
|
||||||
|
import type { ParsedPolicy } from "./parsers/policy-parser";
|
||||||
|
|
||||||
|
function parsed(over: Partial<ParsedPolicy> = {}): ParsedPolicy {
|
||||||
|
return {
|
||||||
|
provider: "GMX",
|
||||||
|
policyNumber: null,
|
||||||
|
insuredName: null,
|
||||||
|
notes: [],
|
||||||
|
coverages: [],
|
||||||
|
vehicles: [],
|
||||||
|
drivers: [],
|
||||||
|
...over,
|
||||||
|
} as unknown as ParsedPolicy;
|
||||||
|
}
|
||||||
|
|
||||||
|
function prismaStub(policies: unknown[], customers: { id: string; name: string }[]) {
|
||||||
|
const findManyPolicy = jest.fn().mockResolvedValue(policies);
|
||||||
|
const findManyCustomer = jest.fn().mockResolvedValue(customers);
|
||||||
|
return {
|
||||||
|
prisma: {
|
||||||
|
policy: { findMany: findManyPolicy },
|
||||||
|
customer: { findMany: findManyCustomer },
|
||||||
|
} as unknown as PrismaService,
|
||||||
|
findManyPolicy,
|
||||||
|
findManyCustomer,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const BOOK = [
|
||||||
|
{ id: "cust-1", name: "WAGONER, PAMELA" },
|
||||||
|
{ id: "cust-2", name: "SMITH, JOHN" },
|
||||||
|
];
|
||||||
|
|
||||||
|
describe("PolicyMatcherService name suggestions", () => {
|
||||||
|
it("suggests a customer when the policy number is new", async () => {
|
||||||
|
const { prisma } = prismaStub([], BOOK);
|
||||||
|
const svc = new PolicyMatcherService(prisma);
|
||||||
|
|
||||||
|
const r = await svc.match(
|
||||||
|
parsed({ policyNumber: "P-999", insuredName: "PAMELA DENISE WAGONER" } as never),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(r.customerSuggestions).toEqual([
|
||||||
|
expect.objectContaining({ customerId: "cust-1", tier: "PARTIAL" }),
|
||||||
|
]);
|
||||||
|
// The suggestion is surfaced, never applied.
|
||||||
|
expect(r.customerId).toBeNull();
|
||||||
|
expect(r.confident).toBe(false);
|
||||||
|
expect(r.note).toContain("posibles clientes por nombre: WAGONER, PAMELA");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("suggests when the policy number could not be read at all", async () => {
|
||||||
|
const { prisma } = prismaStub([], BOOK);
|
||||||
|
const svc = new PolicyMatcherService(prisma);
|
||||||
|
|
||||||
|
const r = await svc.match(parsed({ insuredName: "PAMELA WAGONER" } as never));
|
||||||
|
|
||||||
|
expect(r.customerSuggestions[0]).toMatchObject({ customerId: "cust-1", tier: "EXACT" });
|
||||||
|
expect(r.customerId).toBeNull();
|
||||||
|
expect(r.note).toBe(
|
||||||
|
"no se pudo leer el número de póliza; posible cliente por nombre: WAGONER, PAMELA",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not touch the book when the policy number hits", async () => {
|
||||||
|
const { prisma, findManyCustomer } = prismaStub(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
id: "pol-1",
|
||||||
|
policyNumber: "P-1",
|
||||||
|
customerId: "cust-2",
|
||||||
|
customer: { name: "SMITH, JOHN" },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
BOOK,
|
||||||
|
);
|
||||||
|
const svc = new PolicyMatcherService(prisma);
|
||||||
|
|
||||||
|
const r = await svc.match(
|
||||||
|
parsed({ policyNumber: "P-1", insuredName: "PAMELA WAGONER" } as never),
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(r.confident).toBe(true);
|
||||||
|
expect(r.customerId).toBe("cust-2");
|
||||||
|
expect(r.customerSuggestions).toEqual([]);
|
||||||
|
expect(findManyCustomer).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reads the customer book once across a batch", async () => {
|
||||||
|
const { prisma, findManyCustomer } = prismaStub([], BOOK);
|
||||||
|
const svc = new PolicyMatcherService(prisma);
|
||||||
|
|
||||||
|
await svc.match(parsed({ policyNumber: "A", insuredName: "PAMELA WAGONER" } as never));
|
||||||
|
await svc.match(parsed({ policyNumber: "B", insuredName: "JOHN SMITH" } as never));
|
||||||
|
|
||||||
|
expect(findManyCustomer).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,6 +1,12 @@
|
|||||||
import { Injectable } from "@nestjs/common";
|
import { Injectable } from "@nestjs/common";
|
||||||
import { PrismaService } from "../prisma/prisma.service";
|
import { PrismaService } from "../prisma/prisma.service";
|
||||||
import type { ParsedPolicy } from "./parsers/policy-parser";
|
import type { ParsedPolicy } from "./parsers/policy-parser";
|
||||||
|
import {
|
||||||
|
suggestCustomersByName,
|
||||||
|
suggestionNote,
|
||||||
|
type CustomerNameRow,
|
||||||
|
type CustomerNameSuggestion,
|
||||||
|
} from "./name-matcher";
|
||||||
|
|
||||||
export interface MatchResult {
|
export interface MatchResult {
|
||||||
policyId: string | null;
|
policyId: string | null;
|
||||||
@@ -14,8 +20,24 @@ export interface MatchResult {
|
|||||||
* the policy number is shared across customers and a human must pick.
|
* the policy number is shared across customers and a human must pick.
|
||||||
*/
|
*/
|
||||||
candidates: { policyId: string; customerId: string; customerName: string; policyNumber: string }[];
|
candidates: { policyId: string; customerId: string; customerName: string; policyNumber: string }[];
|
||||||
|
/**
|
||||||
|
* Customers whose name resembles the printed insured name. Populated only
|
||||||
|
* when the policy number resolved to nothing, and never used to set
|
||||||
|
* `customerId` or `confident` — see the class comment.
|
||||||
|
*/
|
||||||
|
customerSuggestions: CustomerNameSuggestion[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How long the customer book is reused across documents in a batch.
|
||||||
|
*
|
||||||
|
* A twenty-page batch would otherwise read all 1536 rows twenty times. The
|
||||||
|
* only cost of the staleness is that a customer created in the last minute
|
||||||
|
* is not suggested — the picker still finds them, so nothing is lost that a
|
||||||
|
* reviewer cannot do in one click.
|
||||||
|
*/
|
||||||
|
const BOOK_TTL_MS = 60_000;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolves a parsed policy page to an existing Policy (and its customer) the
|
* Resolves a parsed policy page to an existing Policy (and its customer) the
|
||||||
* office already holds.
|
* office already holds.
|
||||||
@@ -33,14 +55,29 @@ export interface MatchResult {
|
|||||||
* policy numbers across customers do occur (same group policy bound by two
|
* policy numbers across customers do occur (same group policy bound by two
|
||||||
* related parties), and picking one arbitrarily would silently book the
|
* related parties), and picking one arbitrarily would silently book the
|
||||||
* wrong coverage.
|
* wrong coverage.
|
||||||
|
*
|
||||||
|
* On that zero-hit path only, the printed name is used to *rank the picker*
|
||||||
|
* — see `name-matcher.ts`. That is not a walk-back of the rule above: the
|
||||||
|
* suggestion never reaches `customerId` or `confident`, a human still picks,
|
||||||
|
* and the ranking exists because the office writes names surname-first
|
||||||
|
* ("WAGONER, PAMELA") while carriers print them given-name-first ("PAMELA
|
||||||
|
* DENISE WAGONER"), so the reviewer is retyping a name the machine could
|
||||||
|
* have offered.
|
||||||
*/
|
*/
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class PolicyMatcherService {
|
export class PolicyMatcherService {
|
||||||
|
private book: { rows: CustomerNameRow[]; loadedAt: number } | null = null;
|
||||||
|
|
||||||
constructor(private readonly prisma: PrismaService) {}
|
constructor(private readonly prisma: PrismaService) {}
|
||||||
|
|
||||||
async match(parsed: ParsedPolicy): Promise<MatchResult> {
|
async match(parsed: ParsedPolicy): Promise<MatchResult> {
|
||||||
if (!parsed.policyNumber) {
|
if (!parsed.policyNumber) {
|
||||||
return this.unmatched("no se pudo leer el número de póliza");
|
// No number to search on, so the page goes to review with a picker —
|
||||||
|
// the same place the name suggestions help.
|
||||||
|
return this.unmatched(
|
||||||
|
"no se pudo leer el número de póliza",
|
||||||
|
await this.suggestByName(parsed.insuredName),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const rows = await this.prisma.policy.findMany({
|
const rows = await this.prisma.policy.findMany({
|
||||||
@@ -61,22 +98,33 @@ export class PolicyMatcherService {
|
|||||||
}));
|
}));
|
||||||
|
|
||||||
if (rows.length === 0) {
|
if (rows.length === 0) {
|
||||||
|
const suggestions = await this.suggestByName(parsed.insuredName);
|
||||||
|
const hint = suggestionNote(suggestions);
|
||||||
return {
|
return {
|
||||||
policyId: null,
|
policyId: null,
|
||||||
customerId: null,
|
customerId: null,
|
||||||
note: `no se encontró ninguna póliza con el número ${parsed.policyNumber}`,
|
note: [
|
||||||
|
`no se encontró ninguna póliza con el número ${parsed.policyNumber}`,
|
||||||
|
hint,
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("; "),
|
||||||
confident: false,
|
confident: false,
|
||||||
candidates: [],
|
candidates: [],
|
||||||
|
customerSuggestions: suggestions,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (rows.length > 1) {
|
if (rows.length > 1) {
|
||||||
|
// The policy number did find rows; the reviewer picks among those, and
|
||||||
|
// adding name guesses on top would only add noise.
|
||||||
return {
|
return {
|
||||||
policyId: null,
|
policyId: null,
|
||||||
customerId: null,
|
customerId: null,
|
||||||
note: `${rows.length} pólizas comparten el número ${parsed.policyNumber}`,
|
note: `${rows.length} pólizas comparten el número ${parsed.policyNumber}`,
|
||||||
confident: false,
|
confident: false,
|
||||||
candidates,
|
candidates,
|
||||||
|
customerSuggestions: [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -86,16 +134,47 @@ export class PolicyMatcherService {
|
|||||||
note: `coincidencia exacta por número de póliza ${parsed.policyNumber}`,
|
note: `coincidencia exacta por número de póliza ${parsed.policyNumber}`,
|
||||||
confident: true,
|
confident: true,
|
||||||
candidates,
|
candidates,
|
||||||
|
customerSuggestions: [],
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private unmatched(note: string): MatchResult {
|
private async suggestByName(
|
||||||
|
insuredName: string | null | undefined,
|
||||||
|
): Promise<CustomerNameSuggestion[]> {
|
||||||
|
if (!insuredName) return [];
|
||||||
|
return suggestCustomersByName(insuredName, await this.customerBook());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The whole customer book, held briefly. 1536 rows of `{id, name}` is a
|
||||||
|
* few hundred kilobytes and the comparison is pure token-set work, so
|
||||||
|
* scanning it beats any SQL approximation — and a `LIKE` search would in
|
||||||
|
* any case have to guess which token is the surname, which is the one
|
||||||
|
* thing the office's own data does not agree on.
|
||||||
|
*/
|
||||||
|
private async customerBook(): Promise<CustomerNameRow[]> {
|
||||||
|
if (this.book && Date.now() - this.book.loadedAt < BOOK_TTL_MS) {
|
||||||
|
return this.book.rows;
|
||||||
|
}
|
||||||
|
const rows = await this.prisma.customer.findMany({
|
||||||
|
select: { id: true, name: true },
|
||||||
|
});
|
||||||
|
this.book = { rows, loadedAt: Date.now() };
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
private unmatched(
|
||||||
|
note: string,
|
||||||
|
customerSuggestions: CustomerNameSuggestion[] = [],
|
||||||
|
): MatchResult {
|
||||||
|
const hint = suggestionNote(customerSuggestions);
|
||||||
return {
|
return {
|
||||||
policyId: null,
|
policyId: null,
|
||||||
customerId: null,
|
customerId: null,
|
||||||
note,
|
note: [note, hint].filter(Boolean).join("; "),
|
||||||
confident: false,
|
confident: false,
|
||||||
candidates: [],
|
candidates: [],
|
||||||
|
customerSuggestions,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -211,6 +211,9 @@ export class PolicyOcrService {
|
|||||||
matchCandidates: match.candidates.length
|
matchCandidates: match.candidates.length
|
||||||
? (match.candidates as unknown as Prisma.InputJsonValue)
|
? (match.candidates as unknown as Prisma.InputJsonValue)
|
||||||
: Prisma.DbNull,
|
: Prisma.DbNull,
|
||||||
|
customerSuggestions: match.customerSuggestions.length
|
||||||
|
? (match.customerSuggestions as unknown as Prisma.InputJsonValue)
|
||||||
|
: Prisma.DbNull,
|
||||||
matchNote: notes.join("; "),
|
matchNote: notes.join("; "),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -7,8 +7,13 @@ import { parseBboxLayout } from "./tesseract.provider";
|
|||||||
* that grouping is what left `PERIODO FACTURADO` with no value next to it and
|
* that grouping is what left `PERIODO FACTURADO` with no value next to it and
|
||||||
* every period field empty on a batch whose text was perfectly readable.
|
* every period field empty on a batch whose text was perfectly readable.
|
||||||
*/
|
*/
|
||||||
|
/**
|
||||||
|
* Boxes are sized from the text, at 6 units a character: the reassembler now
|
||||||
|
* reads the space BETWEEN two boxes, so a fixed width would put a fabricated
|
||||||
|
* gap after every short word and every row would come back column-padded.
|
||||||
|
*/
|
||||||
function word(x: number, y: number, text: string): string {
|
function word(x: number, y: number, text: string): string {
|
||||||
return `<word xMin="${x}" yMin="${y}" xMax="${x + 20}" yMax="${y + 8}">${text}</word>`;
|
return `<word xMin="${x}" yMin="${y}" xMax="${x + text.length * 6}" yMax="${y + 8}">${text}</word>`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function doc(...lines: string[]): string {
|
function doc(...lines: string[]): string {
|
||||||
@@ -26,23 +31,57 @@ describe("parseBboxLayout", () => {
|
|||||||
it("rejoins a label with the value printed beside it in another flow", () => {
|
it("rejoins a label with the value printed beside it in another flow", () => {
|
||||||
const [page] = parseBboxLayout(
|
const [page] = parseBboxLayout(
|
||||||
doc(
|
doc(
|
||||||
word(20, 100, "PERIODO") + word(45, 100, "FACTURADO:"),
|
word(20, 100, "PERIODO") + word(68, 100, "FACTURADO:"),
|
||||||
word(300, 100.4, "20260630-20260630"),
|
word(300, 100.4, "20260630-20260630"),
|
||||||
padding(),
|
padding(),
|
||||||
),
|
),
|
||||||
1,
|
1,
|
||||||
);
|
);
|
||||||
expect(page).not.toBeNull();
|
expect(page).not.toBeNull();
|
||||||
expect(page!.text).toContain("PERIODO FACTURADO: 20260630-20260630");
|
expect(page!.text).toMatch(/PERIODO FACTURADO:\s+20260630-20260630/);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps genuinely separate lines apart", () => {
|
it("keeps genuinely separate lines apart", () => {
|
||||||
const [page] = parseBboxLayout(
|
const [page] = parseBboxLayout(
|
||||||
doc(word(20, 100, "Cuenta:") + word(80, 100, "0900003463"), word(20, 130, "Nombre:"), padding()),
|
doc(word(20, 100, "Cuenta:") + word(68, 100, "0900003463"), word(20, 130, "Nombre:"), padding()),
|
||||||
1,
|
1,
|
||||||
);
|
);
|
||||||
expect(page!.text.split("\n")).toContain("Cuenta: 0900003463");
|
const lines = page!.text.split("\n").map((l) => l.trim());
|
||||||
expect(page!.text.split("\n")).toContain("Nombre:");
|
expect(lines).toContain("Cuenta: 0900003463");
|
||||||
|
expect(lines).toContain("Nombre:");
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The layout is data. A borderless table separates its cells with nothing
|
||||||
|
* but white space, so the parsers read a run of spaces as a cell boundary
|
||||||
|
* (`INSURED\s{2,}`) and a column offset as a column (`SUM INSURED` vs
|
||||||
|
* `PREMIUM`). Both regressed to nothing when this collapsed every gap to a
|
||||||
|
* single space, and the fixtures — taken from `pdftotext -layout`, which
|
||||||
|
* prints the gaps — could not see it.
|
||||||
|
*/
|
||||||
|
it("preserves the gap between two cells of a borderless table", () => {
|
||||||
|
const [page] = parseBboxLayout(
|
||||||
|
doc(word(20, 100, "INSURED") + word(300, 100, "PAMELA") + word(340, 100, "WAGONER"), padding()),
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
const line = page!.text.split("\n").find((l) => l.includes("INSURED"))!;
|
||||||
|
expect(line).toMatch(/INSURED\s{2,}PAMELA WAGONER/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves the blank line between two blocks", () => {
|
||||||
|
const [page] = parseBboxLayout(
|
||||||
|
doc(word(20, 100, "Insured"), word(20, 112, "wraps"), word(20, 200, "Next"), padding()),
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
const lines = page!.text.split("\n").map((l) => l.trim());
|
||||||
|
// The wrapped continuation stays attached; the next block is cut off from
|
||||||
|
// it, which is what stops a "join until the cell ends" walk running away.
|
||||||
|
expect(lines.slice(lines.indexOf("Insured"), lines.indexOf("Next") + 1)).toEqual([
|
||||||
|
"Insured",
|
||||||
|
"wraps",
|
||||||
|
"",
|
||||||
|
"Next",
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("scales point coordinates into the render's pixel space", () => {
|
it("scales point coordinates into the render's pixel space", () => {
|
||||||
|
|||||||
@@ -254,6 +254,23 @@ export function parseBboxLayout(xhtml: string, scale: number): (OcrPage | null)[
|
|||||||
* Rows are cut when a word's vertical centre leaves the band established by
|
* Rows are cut when a word's vertical centre leaves the band established by
|
||||||
* the row's first word, which tolerates the sub-pixel baseline differences
|
* the row's first word, which tolerates the sub-pixel baseline differences
|
||||||
* between fonts on one line without merging two genuinely separate lines.
|
* between fonts on one line without merging two genuinely separate lines.
|
||||||
|
*
|
||||||
|
* Vertical WHITE SPACE is preserved as a blank line. Rows alone are not the
|
||||||
|
* whole layout: on a form, the blank between two blocks is what says where a
|
||||||
|
* cell's wrapped value stops, and dropping it leaves parsers that walk a
|
||||||
|
* block ("keep joining until the cell ends") running to the end of the page.
|
||||||
|
* That is not hypothetical — the GMX PVL especificación read its whole first
|
||||||
|
* page as the insured's name, because the fixtures were taken from
|
||||||
|
* `pdftotext -layout` (which prints the blanks) while the runtime fed it this
|
||||||
|
* function's output (which did not).
|
||||||
|
*
|
||||||
|
* Horizontal white space is preserved the same way, by padding each word out
|
||||||
|
* to its own column. The same fixture mismatch bit here: a run of spaces is
|
||||||
|
* the ONLY thing separating two cells of a borderless table, so ANA's
|
||||||
|
* `INSURED\s{2,}` label matches and its `SUM INSURED` / `PREMIUM` column
|
||||||
|
* split (taken from `head.search()` offsets) both need real offsets. Joining
|
||||||
|
* on one space put every driver's-policy premium in the sum-insured column
|
||||||
|
* and left the phone glued to the insured's name.
|
||||||
*/
|
*/
|
||||||
function toVisualRows(words: OcrWord[]): string {
|
function toVisualRows(words: OcrWord[]): string {
|
||||||
const centre = (w: OcrWord) => w.top + w.height / 2;
|
const centre = (w: OcrWord) => w.top + w.height / 2;
|
||||||
@@ -281,14 +298,85 @@ function toVisualRows(words: OcrWord[]): string {
|
|||||||
}
|
}
|
||||||
if (current.length) rows.push(current);
|
if (current.length) rows.push(current);
|
||||||
|
|
||||||
return rows
|
const charWidth = estimateCharWidth(words);
|
||||||
.map((r) =>
|
const out: string[] = [];
|
||||||
[...r]
|
rows.forEach((r, i) => {
|
||||||
.sort((a, b) => a.left - b.left)
|
if (i > 0 && isBlankBetween(rows[i - 1], r)) out.push("");
|
||||||
.map((w) => w.text)
|
out.push(layoutRow(r, charWidth));
|
||||||
.join(" "),
|
});
|
||||||
)
|
return out.join("\n");
|
||||||
.join("\n");
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One row rendered at its printed column offsets.
|
||||||
|
*
|
||||||
|
* Words that merely follow one another inside the same cell are separated by
|
||||||
|
* exactly one space, whatever the column arithmetic says: one `charWidth` for
|
||||||
|
* a page that mixes fonts leaves a rounding error on every word, and letting
|
||||||
|
* that accumulate sprinkles `\s{2,}` runs through ordinary prose — which is
|
||||||
|
* the very thing the parsers read as a cell boundary. Only a gap wide enough
|
||||||
|
* to be deliberate (more than one blank character) is rendered as one, and
|
||||||
|
* only there is the word re-anchored to its true column, so the offsets a
|
||||||
|
* column split depends on stay honest while values stay clean.
|
||||||
|
*/
|
||||||
|
function layoutRow(row: OcrWord[], charWidth: number): string {
|
||||||
|
let line = "";
|
||||||
|
let right = 0;
|
||||||
|
|
||||||
|
for (const w of [...row].sort((a, b) => a.left - b.left)) {
|
||||||
|
const col = Math.round(w.left / charWidth);
|
||||||
|
if (!line.length) {
|
||||||
|
line = " ".repeat(Math.max(0, col));
|
||||||
|
} else if (w.left - right > charWidth * 1.5) {
|
||||||
|
line += " ".repeat(Math.max(2, col - line.length));
|
||||||
|
} else {
|
||||||
|
line += " ";
|
||||||
|
}
|
||||||
|
line += w.text;
|
||||||
|
right = w.left + w.width;
|
||||||
|
}
|
||||||
|
|
||||||
|
return line.trimEnd();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Width of one character, in the same units the word boxes use.
|
||||||
|
*
|
||||||
|
* The median of each word's own width-per-character: robust to the handful of
|
||||||
|
* oversized headings and to the wide-tracked letterhead, both of which would
|
||||||
|
* drag a mean. Only words of 3+ characters vote, since a one-character box is
|
||||||
|
* mostly side bearing. Falls back to a value derived from line height when a
|
||||||
|
* page has nothing long enough to measure.
|
||||||
|
*/
|
||||||
|
function estimateCharWidth(words: OcrWord[]): number {
|
||||||
|
const samples = words
|
||||||
|
.filter((w) => w.text.length >= 3 && w.width > 0)
|
||||||
|
.map((w) => w.width / w.text.length)
|
||||||
|
.sort((a, b) => a - b);
|
||||||
|
if (samples.length) return samples[Math.floor(samples.length / 2)];
|
||||||
|
const heights = words.map((w) => w.height).filter((h) => h > 0);
|
||||||
|
return heights.length ? Math.max(...heights) / 2 : 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Does the space between two consecutive rows read as an empty line?
|
||||||
|
*
|
||||||
|
* Measured against the taller of the two rows so a heading and its body text
|
||||||
|
* are judged on their own scale. On the real documents the two populations do
|
||||||
|
* not overlap: consecutive lines of one paragraph sit at 0.3–1.1 line heights
|
||||||
|
* apart, and anything the reader sees as blank-separated starts at 2.1. The
|
||||||
|
* threshold is placed in that empty middle, biased high — a missed blank only
|
||||||
|
* restores today's behaviour, while a spurious one would cut a wrapped value
|
||||||
|
* short.
|
||||||
|
*/
|
||||||
|
function isBlankBetween(prev: OcrWord[], row: OcrWord[]): boolean {
|
||||||
|
const bottom = Math.max(...prev.map((w) => w.top + w.height));
|
||||||
|
const top = Math.min(...row.map((w) => w.top));
|
||||||
|
const unit = Math.max(
|
||||||
|
...prev.map((w) => w.height),
|
||||||
|
...row.map((w) => w.height),
|
||||||
|
);
|
||||||
|
return unit > 0 && top - bottom > unit * 1.6;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@jorgecuadros/web",
|
"name": "@jorgecuadros/web",
|
||||||
"version": "1.0.19",
|
"version": "1.0.22",
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "next dev -p 4500",
|
"dev": "next dev -p 4500",
|
||||||
|
|||||||
@@ -3114,3 +3114,61 @@ button {
|
|||||||
border-color: var(--brand-500);
|
border-color: var(--brand-500);
|
||||||
color: var(--brand-700);
|
color: var(--brand-700);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ============================================================================
|
||||||
|
Layout + text utilities the screens already assumed
|
||||||
|
Several components were written against these names before any rule
|
||||||
|
defined them, so they rendered as bare inline spans. The visible symptom
|
||||||
|
was the policy OCR review header running together —
|
||||||
|
"Para revisarPágina 1700489616· PAMELA DENISE WAGONERLICENCIASANA" —
|
||||||
|
because JSX drops the newline between sibling elements and the `gap` those
|
||||||
|
call sites pass does nothing without a flex container.
|
||||||
|
========================================================================== */
|
||||||
|
.row {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 0.5rem;
|
||||||
|
}
|
||||||
|
.stack {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 1rem;
|
||||||
|
}
|
||||||
|
/* The muted line under a page title, and the same voice reused inline. Only
|
||||||
|
the block form takes a margin — as a flex child it would shift the item
|
||||||
|
off the row's centre line. */
|
||||||
|
.page-sub {
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 0.875rem;
|
||||||
|
}
|
||||||
|
p.page-sub {
|
||||||
|
margin: 0.25rem 0 0;
|
||||||
|
}
|
||||||
|
/* A neutral chip. Same shape as `.badge` so the OCR statuses, policy type and
|
||||||
|
carrier read as the labels they are rather than as running prose. */
|
||||||
|
.tag {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.375rem;
|
||||||
|
padding: 0.1875rem 0.5625rem;
|
||||||
|
border-radius: 999px;
|
||||||
|
font-size: 0.75rem;
|
||||||
|
font-weight: 600;
|
||||||
|
letter-spacing: 0.01em;
|
||||||
|
line-height: 1.4;
|
||||||
|
white-space: nowrap;
|
||||||
|
background: var(--paper-2);
|
||||||
|
color: var(--muted);
|
||||||
|
border: 1px solid var(--line-strong);
|
||||||
|
}
|
||||||
|
/* The warning sibling of `.state-error`, used where a page needs a human to
|
||||||
|
choose between candidates rather than reporting a failure. */
|
||||||
|
.state-warn {
|
||||||
|
background: var(--servicios-tint);
|
||||||
|
border: 1px solid rgba(154, 106, 18, 0.25);
|
||||||
|
color: var(--servicios-ink);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
padding: 1rem 1.125rem;
|
||||||
|
font-size: 0.875rem;
|
||||||
|
}
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import type {
|
|||||||
PolicyOcrBatchDetail,
|
PolicyOcrBatchDetail,
|
||||||
PolicyOcrConfirmDocument,
|
PolicyOcrConfirmDocument,
|
||||||
PolicyOcrCoverage,
|
PolicyOcrCoverage,
|
||||||
|
PolicyOcrCustomerSuggestion,
|
||||||
PolicyOcrDocument,
|
PolicyOcrDocument,
|
||||||
PolicyOcrReviewInput,
|
PolicyOcrReviewInput,
|
||||||
} from "@/lib/types";
|
} from "@/lib/types";
|
||||||
@@ -180,8 +181,11 @@ export function PolicyOcrReview({ id }: { id: string }) {
|
|||||||
{STATUS_LABEL[batch.status] ?? batch.status}
|
{STATUS_LABEL[batch.status] ?? batch.status}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
<Link className="btn btn-ghost" href="/polizas">
|
{/* Back to the capture screen this batch was uploaded from, not to
|
||||||
Volver a pólizas
|
the policy list — same as the statement review screen, which
|
||||||
|
returns to /recibos. */}
|
||||||
|
<Link className="btn btn-ghost" href="/polizas/captura">
|
||||||
|
Volver a captura
|
||||||
</Link>
|
</Link>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
@@ -351,17 +355,19 @@ function DocumentRow({ doc, customerIndex, canReview, onSave, onReject }: Docume
|
|||||||
const locked = doc.status === "POSTED" || doc.status === "REJECTED";
|
const locked = doc.status === "POSTED" || doc.status === "REJECTED";
|
||||||
const matchedExisting = !!doc.matchedPolicy;
|
const matchedExisting = !!doc.matchedPolicy;
|
||||||
const candidates = doc.matchCandidates ?? [];
|
const candidates = doc.matchCandidates ?? [];
|
||||||
|
const suggestions: PolicyOcrCustomerSuggestion[] = doc.customerSuggestions ?? [];
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<article className="card" style={{ padding: 16 }}>
|
<article className="card" style={{ padding: 16 }}>
|
||||||
<header className="row" style={{ gap: 12, alignItems: "center" }}>
|
<header className="row" style={{ gap: 12, alignItems: "center" }}>
|
||||||
<span className="tag">{STATUS_LABEL[doc.status] ?? doc.status}</span>
|
<span className="tag">{STATUS_LABEL[doc.status] ?? doc.status}</span>
|
||||||
<span className="page-sub">Página {doc.pageNumber}</span>
|
<span className="page-sub">Página {doc.pageNumber}</span>
|
||||||
{doc.extractedPolicyNumber && (
|
{/* No hand-rolled separators or margins here: `.row` is a flex
|
||||||
<strong style={{ marginLeft: 8 }}>{doc.extractedPolicyNumber}</strong>
|
container and its gap does the spacing. A literal "· " would leave
|
||||||
)}
|
a dot floating in that gap. */}
|
||||||
|
{doc.extractedPolicyNumber && <strong>{doc.extractedPolicyNumber}</strong>}
|
||||||
{doc.extractedInsuredName && (
|
{doc.extractedInsuredName && (
|
||||||
<span className="page-sub">· {doc.extractedInsuredName}</span>
|
<span className="page-sub">{doc.extractedInsuredName}</span>
|
||||||
)}
|
)}
|
||||||
{/* Read-only: the parser names the type, the confirm step resolves it
|
{/* Read-only: the parser names the type, the confirm step resolves it
|
||||||
to a policy_types row. Reassigning it is the policy screen's job,
|
to a policy_types row. Reassigning it is the policy screen's job,
|
||||||
@@ -670,6 +676,34 @@ function DocumentRow({ doc, customerIndex, canReview, onSave, onReject }: Docume
|
|||||||
</Field>
|
</Field>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/*
|
||||||
|
* Name suggestions, never a preselection. The office writes
|
||||||
|
* customers surname-first and carriers print them given-name-first,
|
||||||
|
* so without this the reviewer retypes a name the parser already
|
||||||
|
* read. One click fills the picker above; nothing is chosen until
|
||||||
|
* they click. Kept outside the <Field> label — a label must not
|
||||||
|
* wrap other interactive controls.
|
||||||
|
*/}
|
||||||
|
{!policyId && !customerId && !locked && canReview && suggestions.length > 0 && (
|
||||||
|
<div className="row" style={{ gap: 8, flexWrap: "wrap" }}>
|
||||||
|
<span className="page-sub">Sugerencias por nombre:</span>
|
||||||
|
{suggestions.map((s) => (
|
||||||
|
<button
|
||||||
|
key={s.customerId}
|
||||||
|
type="button"
|
||||||
|
className="btn btn-ghost btn-sm"
|
||||||
|
onClick={() => {
|
||||||
|
setCustomerId(s.customerId);
|
||||||
|
setCustomerName(s.customerName);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{s.customerName}
|
||||||
|
{s.tier === "PARTIAL" && <span className="page-sub"> · parcial</span>}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<label className="field">
|
<label className="field">
|
||||||
<input
|
<input
|
||||||
type="checkbox"
|
type="checkbox"
|
||||||
|
|||||||
@@ -1477,6 +1477,18 @@ export interface PolicyOcrMatchCandidate {
|
|||||||
policyNumber: string;
|
policyNumber: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A customer whose name resembles the printed insured name. A suggestion,
|
||||||
|
* not a match — the API never preselects one.
|
||||||
|
*/
|
||||||
|
export interface PolicyOcrCustomerSuggestion {
|
||||||
|
customerId: string;
|
||||||
|
customerName: string;
|
||||||
|
/** `EXACT` = same name tokens in any order; `PARTIAL` = one contains the other. */
|
||||||
|
tier: "EXACT" | "PARTIAL";
|
||||||
|
score: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface PolicyOcrDocument {
|
export interface PolicyOcrDocument {
|
||||||
id: string;
|
id: string;
|
||||||
pageNumber: number;
|
pageNumber: number;
|
||||||
@@ -1512,6 +1524,7 @@ export interface PolicyOcrDocument {
|
|||||||
} | null;
|
} | null;
|
||||||
matchedCustomer: { id: string; name: string } | null;
|
matchedCustomer: { id: string; name: string } | null;
|
||||||
matchCandidates: PolicyOcrMatchCandidate[] | null;
|
matchCandidates: PolicyOcrMatchCandidate[] | null;
|
||||||
|
customerSuggestions: PolicyOcrCustomerSuggestion[] | null;
|
||||||
matchNote: string | null;
|
matchNote: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -134,6 +134,55 @@ customers do occur (one group policy bound by two related parties), and
|
|||||||
picking arbitrarily would silently book the wrong coverage against the wrong
|
picking arbitrarily would silently book the wrong coverage against the wrong
|
||||||
person.
|
person.
|
||||||
|
|
||||||
|
### Name suggestions on the zero-hit path
|
||||||
|
|
||||||
|
When the policy number finds nothing — the new-policy case, where a human has
|
||||||
|
to pick a customer anyway — `name-matcher.ts` ranks the customer book against
|
||||||
|
the printed insured name and the review screen offers the top three as
|
||||||
|
one-click buttons above the picker. They are written to
|
||||||
|
`policy_ocr_documents.customerSuggestions`, deliberately **not** to
|
||||||
|
`matchCandidates`, so a name hint can never be read as a policy-number hit.
|
||||||
|
Nothing sets `matchedCustomerId`; the rule above is unchanged.
|
||||||
|
|
||||||
|
The problem is only ordering: the office books customers surname-first
|
||||||
|
(`WAGONER, PAMELA`) and carriers print them given-name-first
|
||||||
|
(`PAMELA DENISE WAGONER`), so a string compare never hits while a **token-set**
|
||||||
|
compare does. Names are normalized (accents folded, so OCR's `MUNOZ` reaches
|
||||||
|
the book's `MUÑOZ`; initials, `DE`/`LA`/`Y`, `JR`, `S.A. DE C.V.` and any token
|
||||||
|
containing a digit dropped — ANA prints the phone hard against the name as
|
||||||
|
`Ph.3102001538`). Two tiers:
|
||||||
|
|
||||||
|
| Tier | Rule |
|
||||||
|
|---|---|
|
||||||
|
| `EXACT` | identical token sets, any order |
|
||||||
|
| `PARTIAL` | one set contains the other, ≥2 shared tokens, **and** the surname is present |
|
||||||
|
|
||||||
|
Both thresholds come from measuring the real book (1536 customers):
|
||||||
|
|
||||||
|
- 1487 distinct token sets, so `EXACT` cross-person collisions are ~0
|
||||||
|
- loosen to surname + first given name and 131 customers (8.5%) collide —
|
||||||
|
the book holds `MCWILLIAMS, BRIAN MICHAEL` *and* `MCWILLIAMS, BRIAN`
|
||||||
|
- 185 surnames are shared by 524 customers, so one token is never evidence;
|
||||||
|
hence the ≥2 floor and the explicit surname requirement, which is what stops
|
||||||
|
`JERRY MARILYN` reaching `ESTRADA, JERRY & MARILYN` on given names alone
|
||||||
|
|
||||||
|
Replaying every book row as a carrier would print it (given-name-first, joint
|
||||||
|
spouse dropped): 97.9% top-ranked correct, 0.9% no suggestion, 1.2% a
|
||||||
|
different row — and all but two of those are the same human on a duplicate or
|
||||||
|
variant row (`MOLNAR, JANOS` vs `MOLNAR, JANOS`, `IBARRA, ISMAEL &`). The
|
||||||
|
two genuine wrong-person cases are `CUADROS, JORGE JR` against three
|
||||||
|
`CUADROS, JORGE H.`, and they appear as a tie in the list rather than as a
|
||||||
|
single answer.
|
||||||
|
|
||||||
|
A blob is refused outright (>8 tokens or >80 characters): GMX's especificación
|
||||||
|
has no field labels and the parser has been seen handing its whole first page
|
||||||
|
over as `insuredName`, which would find a surname somewhere in the prose.
|
||||||
|
`(SIN NOMBRE)` — 14 rows the migration left — is skipped on both sides.
|
||||||
|
|
||||||
|
**Not used for utility statements.** There the registrant genuinely is not the
|
||||||
|
customer (the `CATT, RANDY` finding above), so the same trick would be wrong,
|
||||||
|
not merely noisy.
|
||||||
|
|
||||||
## GMX ships two unrelated documents for the same policy
|
## GMX ships two unrelated documents for the same policy
|
||||||
|
|
||||||
The office downloads both from the same portal, and either can land in a
|
The office downloads both from the same portal, and either can land in a
|
||||||
@@ -457,6 +506,17 @@ term, the excluded sections, the column-position split on the driver's policy,
|
|||||||
its different section order, and — for both faces — that a doubled or tripled
|
its different section order, and — for both faces — that a doubled or tripled
|
||||||
input yields one set of coverages and one driver rather than one per copy.
|
input yields one set of coverages and one driver rather than one per copy.
|
||||||
|
|
||||||
|
`apps/api/src/policy-ocr/name-matcher.spec.ts` — 21 cases on the customer name
|
||||||
|
suggestions, every fixture name lifted from the real book: the reversed name,
|
||||||
|
the printed middle name, the exact row outranking the row that merely contains
|
||||||
|
it, the joint account reached from one spouse (and refused when only given
|
||||||
|
names are printed), the Spanish double surname with the comma in either place,
|
||||||
|
the 54 rows with no comma at all, the `(SIN NOMBRE)` placeholder, a bare shared
|
||||||
|
surname, and the page-sized blob. Four more in
|
||||||
|
`policy-matcher.service.spec.ts` pin the wiring: suggestions on the zero-hit
|
||||||
|
and unreadable-number paths, no book read at all when the policy number hits,
|
||||||
|
and one book read across a batch.
|
||||||
|
|
||||||
Four of the GMX cases are regression tests for ways the parser can silently attach
|
Four of the GMX cases are regression tests for ways the parser can silently attach
|
||||||
the *wrong* value rather than none — a neighbouring coverage's prose read as
|
the *wrong* value rather than none — a neighbouring coverage's prose read as
|
||||||
a deductible, the page-level `DEDUCIBLES:` paragraph read as one, a coverage
|
a deductible, the page-level `DEDUCIBLES:` paragraph read as one, a coverage
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "jorgecuadros-platform",
|
"name": "jorgecuadros-platform",
|
||||||
"version": "1.0.19",
|
"version": "1.0.22",
|
||||||
"private": true,
|
"private": true,
|
||||||
"workspaces": [
|
"workspaces": [
|
||||||
"apps/*",
|
"apps/*",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@jorgecuadros/database",
|
"name": "@jorgecuadros/database",
|
||||||
"version": "1.0.19",
|
"version": "1.0.22",
|
||||||
"private": true,
|
"private": true,
|
||||||
"main": "generated/client/index.js",
|
"main": "generated/client/index.js",
|
||||||
"types": "generated/client/index.d.ts",
|
"types": "generated/client/index.d.ts",
|
||||||
|
|||||||
+7
@@ -0,0 +1,7 @@
|
|||||||
|
-- Ranked customers whose name matches the printed insured name, for the
|
||||||
|
-- documents whose policy number found nothing and therefore need a customer
|
||||||
|
-- picked by hand. Kept in its own column rather than folded into
|
||||||
|
-- `matchCandidates`, which the review screen reads as policy-number hits —
|
||||||
|
-- a name is a suggestion and must never be able to masquerade as a match.
|
||||||
|
ALTER TABLE `policy_ocr_documents`
|
||||||
|
ADD COLUMN `customerSuggestions` JSON NULL;
|
||||||
@@ -476,6 +476,13 @@ model PolicyOcrDocument {
|
|||||||
/// normal; >1 means the policy number is shared across customers and a
|
/// normal; >1 means the policy number is shared across customers and a
|
||||||
/// human must pick.
|
/// human must pick.
|
||||||
matchCandidates Json?
|
matchCandidates Json?
|
||||||
|
/// `CustomerNameSuggestion[]` — customers whose name matches the printed
|
||||||
|
/// insured name, ranked. A SUGGESTION, never a match: it is deliberately
|
||||||
|
/// kept out of `matchCandidates` so the review screen cannot mistake a
|
||||||
|
/// name hint for a policy-number hit, and it never sets
|
||||||
|
/// `matchedCustomerId`. Only populated when the policy number found
|
||||||
|
/// nothing, which is exactly when staff have to pick a customer by hand.
|
||||||
|
customerSuggestions Json?
|
||||||
/// Text, not VARCHAR(191): this carries the parser's whole note trail, and
|
/// Text, not VARCHAR(191): this carries the parser's whole note trail, and
|
||||||
/// a multi-section ANA policy runs past 191 characters routinely. Silently
|
/// a multi-section ANA policy runs past 191 characters routinely. Silently
|
||||||
/// truncating it drops the tail notes, which are the ones that say what
|
/// truncating it drops the tail notes, which are the ones that say what
|
||||||
|
|||||||
Reference in New Issue
Block a user