diff --git a/apps/api/src/policy-ocr/parsers/policy-parser.spec.ts b/apps/api/src/policy-ocr/parsers/policy-parser.spec.ts index fb83071..ada16a2 100644 --- a/apps/api/src/policy-ocr/parsers/policy-parser.spec.ts +++ b/apps/api/src/policy-ocr/parsers/policy-parser.spec.ts @@ -144,4 +144,265 @@ describe("parsePolicy / GMX", () => { expect(eqTyped.deductible).toContain("sum insured"); expect(eqTyped.lossParticipation).toBe("20%"); }); +}); + +/** + * The second GMX document family: the Spanish PVL "especificación" the office + * receives as `…-CondicionesParticulares.pdf`. Verbatim excerpts from + * `007_LGS-HGMX_07006957_01_0-CondicionesParticulares.pdf` through + * `pdftotext -layout`, indentation included — the column positions and the + * blank lines between blocks are what the parser reads, so a cleaned-up + * fixture would test nothing. + */ +describe("parsePolicy / GMX especificación (PVL Hogar)", () => { + const HEADER = + " ESPECIFICACIÓN QUE SE ADHIERE Y FORMA PARTE INTEGRANTE DE LA PÓLIZA\n" + + " 07-037-07006957-00000-01\n" + + "\n"; + + const GMX_ESPEC = page( + HEADER + + "\n" + + " Nombre del asegurado EMMER . KATHLEEN\n" + + "\n" + + " Tipo Persona Asegurada Propietario\n" + + "\n" + + " Ubicación del riesgo LOS PELICANOS ESTE NO. 98 Col. LAS GAVIOTAS PLAYAS\n" + + " DE ROSARITO BAJA CALIFORNIA 22713\n" + + "\n" + + " Características del Inmueble Casa Tipo constructivo Combinado: Macizo y Madera.\n" + + " Consta de 2 pisos incluyendo sótanos y planta baja.\n" + + "\n" + + " -500 mts.cuerpo agua SI\n" + + "\n" + + " Asegurado Adicional\n" + + "\n" + + "PVL Hogar - GMX Seguros Página: 1 de 10\n" + + HEADER + + "\n" + + " SECCIÓN INCENDIO EDIFICIO Y CONTENIDOS\n" + + "\n" + + " EDIFICIO\n" + + "\n" + + " Límite Máximo de Responsabilidad:\n" + + " $200,000.00 USD\n" + + "\n" + + " Quedan amparados los muros de contención y bardas, así como puertas y portones, hasta un sublimite de $ 50,000.00 M.N. o su\n" + + " equivalente en dólares americanos, o hasta el 10% de la suma asegurada de la sección de Edificio, lo que resulte menor.\n" + + "\n" + + "\n" + + " CONTENIDOS\n" + + "\n" + + " Límite Máximo de Responsabilidad:\n" + + " $20,000.00 USD\n" + + "\n" + + " 2. Terremoto o erupción volcánica: Sección Edificio EXCLUIDO, Sección Contenidos EXCLUIDO\n" + + "\n" + + " 3. Fenómenos hidrometeorológicos: Sección Edificio $200,000.00 USD, Sección Contenidos $20,000.00 USD\n" + + "\n" + + " Riesgos adicionales.\n" + + "\n" + + " Remoción de escombros\n" + + "\n" + + " Límite Máximo de Responsabilidad:\n" + + " Edificio\n" + + " $20,000.00 USD\n" + + " Contenidos\n" + + " $2,000.00 USD\n" + + "\n" + + " Gastos extraordinarios para casa habitación\n" + + "\n" + + " En caso de siniestro por los riesgos cubiertos en esta póliza, GMX Seguros pagará la renta de casa o departamento, casa de\n" + + " huéspedes u hotel cuando se asegure el inmueble, así como los gastos de mudanza, seguro de transporte del menaje de casa y\n" + + " efectuados.\n" + + "\n" + + " Límite Máximo de Responsabilidad:\n" + + " $22,000.00 USD\n" + + " Periodo de indemnización: 4 meses.\n" + + "\n" + + " Bienes a la Intemperie:\n" + + "\n" + + "\n" + + " 5 POR CIENTO SOBRE SUMA ASEGURADA, 20 PORCIENTO DE PARTICIPACIÓN A CARGO DEL ASEGURADO DE TODA\n" + + " Y CADA PÉRDIDA.\n" + + "\n" + + "\n" + + " Límite Máximo de Responsabilidad: $10,000.00 USD\n" + + "\n" + + " DEDUCIBLES:\n" + + "\n" + + " El procedimiento que se seguirá para la aplicación de deducibles en caso de que la póliza cuente con cláusula inflacionaria en todas\n" + + " y/o en algunas de sus coberturas será como sigue:\n" + + "\n" + + " Fenómenos hidrometeorológicos\n" + + " Zona: A2\n" + + " Deducible\n" + + " Edificio: 1 POR CIENTO SOBRE SUMA ASEGURADA\n" + + " Coaseguro:\n" + + " Zona 1: (INTERIOR) Participación a cargo del asegurado del 10% de toda y cada pérdida.\n" + + " Zona 2: Participación a cargo del asegurado del 10% de toda y cada pérdida.\n" + + "\n" + + " Deducible\n" + + " Contenidos: 1 POR CIENTO SOBRE SUMA ASEGURADA\n" + + "\n" + + " II.- SECCIÓN DIVERSOS MISCELÁNEOS\n" + + "\n" + + " ROBO DE CONTENIDOS\n" + + "\n" + + " Límite de Responsabilidad:\n" + + " $4,000.00 USD\n" + + "\n" + + "\n" + + " Deducible:\n" + + " Sin deducible\n" + + "\n" + + "\n" + + " Sublímites:\n" + + " Joyas, artículos de oro y plata, armas, relojes, pieles, piedras preciosas montadas, colecciones, obras de arte y demás que por su\n" + + "\n" + + "PVL Hogar - GMX Seguros Página: 7 de 10\n" + + HEADER + + "\n" + + " naturaleza se consideran como objetos de difícil o imposible reposición\n" + + "\n" + + "\n" + + " Límite de Responsabilidad:\n" + + "\n" + + " $2,000.00 USD\n" + + "\n" + + "\n" + + " Deducible:\n" + + " Sin deducible\n" + + "\n" + + " Las condiciones generales que forman parte de la presente póliza son las identificadas bajo el nombre:\n" + + " W_HogarGMX_12.11.2025.pdf\n" + + "\n" + + "PVL Hogar - GMX Seguros Página: 10 de 10\n", + ); + + const byRisk = (p: ReturnType) => + Object.fromEntries(p.coverages.map((c) => [c.risk, c])); + + it("reads a policy number whose groups are not the caratula's widths", () => { + // 2-3-8-5-2 here vs 3-3-8-4-2 on the English caratula. Pinning the widths + // reads one family and returns null on the other. + expect(parsePolicy(GMX_ESPEC).policyNumber).toBe("07-037-07006957-00000-01"); + }); + + it("reads the insured, the risk location across its wrapped line, and the ZIP", () => { + const p = parsePolicy(GMX_ESPEC); + expect(p.provider).toBe("GMX"); + expect(p.insuredName).toBe("EMMER . KATHLEEN"); + expect(p.legalAddress).toBe( + "LOS PELICANOS ESTE NO. 98 Col. LAS GAVIOTAS PLAYAS DE ROSARITO BAJA CALIFORNIA 22713", + ); + expect(p.zip).toBe("22713"); + // The cell is printed but empty on this policy — an empty label must not + // capture the next line of the form. + expect(p.additionalInsured).toBeNull(); + }); + + it("leaves the fields this document does not carry null, and says so", () => { + const p = parsePolicy(GMX_ESPEC); + expect(p.policyFrom).toBeNull(); + expect(p.policyTo).toBeNull(); + expect(p.policyDate).toBeNull(); + expect(p.agentName).toBeNull(); + expect(p.netPremium).toBeNull(); + expect(p.total).toBeNull(); + expect(p.notes.join(" ")).toMatch(/no trae vigencia, agente ni prima/i); + }); + + it("takes the currency from the printed limits, not from the M.N. sublimits", () => { + // The body prose quotes sublimits in pesos ("$ 50,000.00 M.N."); every + // limit is in USD, and only the limits vote. + expect(parsePolicy(GMX_ESPEC).currency).toBe("USD"); + }); + + it("reads each coverage under its own heading", () => { + const c = byRisk(parsePolicy(GMX_ESPEC)); + expect(c.EDIFICIO?.insuredAmount).toBe(200000); + expect(c.CONTENIDOS?.insuredAmount).toBe(20000); + expect(c["ROBO DE CONTENIDOS"]?.insuredAmount).toBe(4000); + expect(c["ROBO DE CONTENIDOS"]?.deductible).toBe("Sin deducible"); + }); + + it("splits a limit printed under Edificio / Contenidos sub-labels", () => { + const c = byRisk(parsePolicy(GMX_ESPEC)); + expect(c["Remoción de escombros — Edificio"]?.insuredAmount).toBe(20000); + expect(c["Remoción de escombros — Contenidos"]?.insuredAmount).toBe(2000); + }); + + it("names a coverage after its heading, not after the wrapped tail of the prose above it", () => { + // Walking back from the limit hits "efectuados." — short, and the only + // thing separating it from a heading is that it is not preceded by a + // blank line. + const c = byRisk(parsePolicy(GMX_ESPEC)); + expect(c["Gastos extraordinarios para casa habitación"]?.insuredAmount).toBe(22000); + expect(c["efectuados."]).toBeUndefined(); + }); + + it("reads a limit printed on the label's own line", () => { + const c = byRisk(parsePolicy(GMX_ESPEC)); + expect(c["Bienes a la Intemperie"]?.insuredAmount).toBe(10000); + }); + + it("reads a deductible stated as a sentence above the limit", () => { + const c = byRisk(parsePolicy(GMX_ESPEC)); + expect(c["Bienes a la Intemperie"]?.deductible).toBe( + "5 POR CIENTO SOBRE SUMA ASEGURADA, 20 PORCIENTO DE PARTICIPACIÓN A CARGO DEL ASEGURADO DE TODA Y CADA PÉRDIDA.", + ); + }); + + it("never borrows a neighbouring coverage's prose as a deductible", () => { + // "…o hasta el 10% de la suma asegurada de la sección de Edificio" is a + // sublimit rule for EDIFICIO, printed two paragraphs above CONTENIDOS. + const c = byRisk(parsePolicy(GMX_ESPEC)); + expect(c.CONTENIDOS?.deductible).toBeNull(); + expect(c.EDIFICIO?.deductible).toBeNull(); + }); + + it("does not read the page-level DEDUCIBLES paragraph as a deductible", () => { + const p = parsePolicy(GMX_ESPEC); + expect( + p.coverages.some((c) => (c.deductible ?? "").includes("cláusula inflacionaria")), + ).toBe(false); + }); + + it("reads a sublimit block as a sublimit OF the coverage above it", () => { + // The amount sits after a blank line AND a page break, and the block's + // own heading ("Sublímites:") names no risk. + const c = byRisk(parsePolicy(GMX_ESPEC)); + expect(c["ROBO DE CONTENIDOS — sublímite"]?.insuredAmount).toBe(2000); + }); + + it("records an excluded catastrophic risk as excluded, never as zero", () => { + const p = parsePolicy(GMX_ESPEC); + const quake = p.coverages.filter((c) => /Terremoto/i.test(c.risk)); + expect(quake).toHaveLength(2); + for (const c of quake) { + expect(c.risk).toMatch(/EXCLUIDO/); + // A coverage insured for $0 and an excluded coverage are the same + // number and very different facts. + expect(c.insuredAmount).toBeNull(); + } + }); + + it("attaches the hydrometeorological deductible and coinsurance from its own block", () => { + const c = byRisk(parsePolicy(GMX_ESPEC)); + const building = c["Fenómenos hidrometeorológicos — Sección Edificio"]; + expect(building?.insuredAmount).toBe(200000); + expect(building?.deductible).toBe("1 POR CIENTO SOBRE SUMA ASEGURADA"); + expect(building?.lossParticipation).toBe("10%"); + expect(c["Fenómenos hidrometeorológicos — Sección Contenidos"]?.insuredAmount).toBe(20000); + }); + + it("carries the underwriting context the fields have no home for", () => { + const notes = parsePolicy(GMX_ESPEC).notes.join(" | "); + expect(notes).toMatch(/tipo de persona asegurada: Propietario/); + expect(notes).toMatch(/características del inmueble: Casa/); + expect(notes).toMatch(/cuerpo de agua/); + expect(notes).toMatch(/zona catastrófica declarada: A2/); + expect(notes).toMatch(/W_HogarGMX_12\.11\.2025\.pdf/); + }); }); \ No newline at end of file diff --git a/apps/api/src/policy-ocr/parsers/policy-parser.ts b/apps/api/src/policy-ocr/parsers/policy-parser.ts index 1b2a8dc..17b6055 100644 --- a/apps/api/src/policy-ocr/parsers/policy-parser.ts +++ b/apps/api/src/policy-ocr/parsers/policy-parser.ts @@ -100,6 +100,50 @@ function firstMatch(text: string, patterns: RegExp[]): string | null { return null; } +function allMatches(text: string, pattern: RegExp): string[] { + const re = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`); + const out: string[] = []; + let m: RegExpExecArray | null; + while ((m = re.exec(text)) !== null) { + if (m[1]) out.push(m[1]); + if (m.index === re.lastIndex) re.lastIndex++; + } + return out; +} + +/** + * The printed policy number, as a shape rather than as fixed widths. + * + * Both GMX document families print the same five dash-separated groups, but + * NOT with the same widths: the English caratula reads + * `007-037-07005947-0000-02` and the Spanish PVL especificación reads + * `07-037-07006957-00000-01` — 2 digits in the first group and 5 in the + * fourth. Pinning the widths (as the first version of this parser did) reads + * one family and silently returns null on the other, which the matcher then + * reports as "no se pudo leer el número de póliza" on a page where the number + * is printed ten times. + * + * The OCR confusion class is allowed inside the groups but NOT normalised + * here — `normalizePolicyNumber` does that, so the raw read stays available. + */ +const POLICY_NUMBER_SHAPE = + /\b([0-9OIlSBD]{2,4}[-\s][0-9OIlSBD]{3}[-\s][0-9OIlSBD]{6,10}[-\s][0-9OIlSBD]{4,6}[-\s][0-9OIlSBD]{2})\b/; + +/** + * Strip layout whitespace and fold the Tesseract digit confusions, keeping the + * dashes: they are part of the number as the office keys it, and + * `PolicyMatcherService` matches `Policy.policyNumber` exactly. + */ +function normalizePolicyNumber(raw: string | null | undefined): string | null { + if (!raw) return null; + const out = raw + .replace(/\s+/g, "") + .split("") + .map((c) => (c === "-" ? c : (DIGIT_CONFUSIONS[c] ?? c))) + .join(""); + return out || null; +} + const MONTHS: Record = { ENE: 0, FEB: 1, MAR: 2, ABR: 3, MAY: 4, JUN: 5, JUL: 6, AGO: 7, SEP: 8, OCT: 9, NOV: 10, DIC: 11, @@ -170,7 +214,10 @@ const BRAND: [string, RegExp][] = [ ]; const LAYOUT: [string, RegExp][] = [ - ["GMX", /Multiple\s*Policy|IMPUESTO\s*PREDIAL[\s\S]{0,80}EN\s*FECHA|Material\s*damages\s*Section/i], + [ + "GMX", + /Multiple\s*Policy|IMPUESTO\s*PREDIAL[\s\S]{0,80}EN\s*FECHA|Material\s*damages\s*Section|PVL\s*Hogar|ESPECIFICACI[ÓO]N\s*QUE\s*SE\s*ADHIERE/i, + ], ]; export function detectPolicyProvider(text: string): string | null { @@ -224,6 +271,33 @@ export function parsePolicy(page: OcrPage): ParsedPolicy { // --- GMX -------------------------------------------------------------------- +/** + * GMX ships (at least) two unrelated document families, and the office + * downloads both from the same portal for the same policy: + * + * 1. **Caratula / translation** (`…_Traduccion.pdf`) — the English-labelled + * certificate: a boxed header table (Policy / Insured / Broker / From / To + * / Currency) and a four-column coverage table. + * 2. **Especificación** (`…-CondicionesParticulares.pdf`) — the Spanish + * "ESPECIFICACIÓN QUE SE ADHIERE Y FORMA PARTE INTEGRANTE DE LA PÓLIZA" + * issued by GMX's Punto de Venta en Línea (PVL). Ten pages of prose with + * NO header table, NO dates, NO broker, and the coverages expressed as + * section headings above a `Límite Máximo de Responsabilidad:` label. + * + * They share only the brand and the policy number, so they get separate + * parsers behind one provider. Both still return `provider: "GMX"` — the + * matcher keys on the policy number alone and must not care which artifact + * the office happened to upload. + */ +function parseGmx(page: OcrPage): ParsedPolicy { + return isEspecificacion(page.text) ? parseGmxEspecificacion(page) : parseGmxCaratula(page); +} + +/** The PVL especificación announces itself in its own repeated page header. */ +function isEspecificacion(text: string): boolean { + return /ESPECIFICACI[ÓO]N\s+QUE\s+SE\s+ADHIERE|PVL\s+Hogar|Nombre\s+del\s+asegurado/i.test(text); +} + /** * GMX policy certificate layout (this is the translation PDF — the Spanish * version is the canonical source, but every GMX portal download is a @@ -244,7 +318,7 @@ export function parsePolicy(page: OcrPage): ParsedPolicy { * certificate alone, and the staff confirm step fills premium in by hand * or after a follow-up receipt upload. */ -function parseGmx(page: OcrPage): ParsedPolicy { +function parseGmxCaratula(page: OcrPage): ParsedPolicy { const text = page.text; const notes: string[] = []; @@ -255,7 +329,7 @@ function parseGmx(page: OcrPage): ParsedPolicy { // never looks like one. The dashes are part of the printed number — keep // them (don't run toDigits, which would flatten them). const policyNumber = firstMatch(text, [ - /\bPolicy\s+([0-9OIlSBD]{3,4}[-\s][0-9OIlSBD]{3}[-\s][0-9OIlSBD]{8}[-\s][0-9OIlSBD]{4}[-\s][0-9OIlSBD]{2})/i, + new RegExp(`\\bPolicy\\s+${POLICY_NUMBER_SHAPE.source}`, "i"), /\bPolicy\s+([0-9OIlSBD][0-9OIlSBD\s-]{9,30})/, ]); @@ -267,12 +341,15 @@ function parseGmx(page: OcrPage): ParsedPolicy { // Legal address is a single long line; the parser keeps it whole. const legalAddress = labelValue(text, /^Legal\s+address\s+(.+)$/m); - const zip = labelValue(text, /^ZIP\s+(\d{4,6})\b/m); + let zip = labelValue(text, /^ZIP\s+(\d{4,6})\b/m); if (!zip && legalAddress) { // Last resort: zip often appears at the tail of the address run too // ("…C.P. 22550"). Cheap regex, no false-positive cost on this layout. - const m = legalAddress.match(/\b(\d{5})\b/); - if (m) notes.push(`ZIP leído de la dirección (${m[1]})`); + const m = legalAddress.match(/\b(\d{5})\b(?!.*\b\d{5}\b)/); + if (m) { + zip = m[1]; + notes.push(`ZIP leído de la dirección (${m[1]})`); + } } // Broker line on GMX: "(1176) Jorge Humberto Cuadros" — the number is the @@ -323,7 +400,7 @@ function parseGmx(page: OcrPage): ParsedPolicy { return { provider: "GMX", - policyNumber: policyNumber ? policyNumber.replace(/\s+/g, "") : null, + policyNumber: normalizePolicyNumber(policyNumber), insuredName, additionalInsured, agentName, @@ -419,4 +496,525 @@ function parseGmxCoverages(text: string, notes: string[]): ParsedCoverage[] { if (out.length === 0) notes.push("no se encontraron coberturas en la tabla"); return out; +} + +// --- GMX / PVL especificación ----------------------------------------------- + +/** + * Page furniture on the PVL especificación: the two-line header that repeats + * on all ten pages (title + policy number on its own line) and the footer. + * Skipped when hunting for a coverage's heading, or the heading found would + * be the page break rather than the risk. + */ +const ESPEC_FURNITURE = + /^(ESPECIFICACI[ÓO]N\s+QUE\s+SE\s+ADHIERE|PVL\s+Hogar\s*-\s*GMX|P[áa]gina\s*:|La\s+presente\s+p[óo]liza\s+queda\s+sujeta)/i; + +/** + * Labels that structure a coverage block rather than name one. A heading walk + * that accepts these ends up calling every coverage "Deducible:". + */ +const ESPEC_STRUCTURAL = + /^(L[íi]mites?\b|Deducibles?\b|Coaseguros?\b|Bienes\s+cubiertos|Riesgos\s+cubiertos|Riesgos\s+adicionales|Exclusiones?\b|Zona\s*:|Periodo\s+de\s+indemnizaci[óo]n|Nota\s+aclaratoria|CONDICIONES\s+ESPECIALES|IMPORTANTE\b|SECCI[ÓO]N\b|II+\.-|[●•]|\d+\.\s)/i; + +/** `$200,000.00 USD`, `$ 35,050.00 M.N.` — amount plus its printed currency. */ +const ESPEC_AMOUNT = /\$\s*([\d,]+(?:\.\d{2})?)\s*(USD|MXN|M\.?\s?N\.?|D[LlÓó]{1,3}[Ss]?)?/; + +/** The label that introduces every insured amount on this layout. */ +const ESPEC_LIMIT_LABEL = /^\s*L[íi]mite\s+(?:M[áa]ximo\s+)?(?:de\s+)?Responsabilidad\s*:?(.*)$/i; + +/** + * GMX's PVL "especificación" layout — ten pages of Spanish prose that the + * office receives as `…-CondicionesParticulares.pdf`. + * + * What this document DOES carry: the policy number (printed on every page + * header), the insured's name, the risk location, and every coverage limit. + * + * What it does NOT carry, at all: policy dates, broker, premium, currency as + * a labelled field, or a coverage table. There is nothing to read for those, + * so they stay null and the notes say why — the caratula is the document that + * has them, and the office downloads both for the same policy. + * + * Coverages are not tabular here. Each one is a section heading followed by + * `Límite Máximo de Responsabilidad:` and an amount on the next line (or two + * amounts under `Edificio` / `Contenidos` sub-labels), with the deductible + * printed further down its own block. So the parser anchors on the limit + * label and walks BACKWARDS for the heading, rather than trying to match a + * row shape that does not exist. + */ +function parseGmxEspecificacion(page: OcrPage): ParsedPolicy { + const text = page.text; + const lines = text.split("\n"); + const notes: string[] = []; + + // ----- policy number ----------------------------------------------------- + // Printed on its own line under the title on all ten pages. Ten independent + // readings of the same number is a free cross-check: on a scan, a group that + // OCR'd two different ways means one reading is wrong and there is no third + // to break the tie, so the row goes to a human (the same rule the zona + // federal statement parser applies to its clave). + const readings = [ + ...new Set( + allMatches(text, POLICY_NUMBER_SHAPE) + .map(normalizePolicyNumber) + .filter((n): n is string => n != null), + ), + ]; + const policyNumber = readings[0] ?? null; + if (!policyNumber) notes.push("no se pudo leer el número de póliza"); + if (readings.length > 1) { + notes.push( + `el número de póliza se leyó de ${readings.length} formas distintas (${readings.join(", ")})`, + ); + } + + // ----- insured + risk location ------------------------------------------ + // Values wrap onto the following lines within the same visual cell, and a + // blank line always ends the cell, so the block reader joins until one. + const insuredName = espectBlock(lines, /^\s*Nombre\s+del\s+asegurado\s+(.+)$/i); + const additionalInsured = espectBlock(lines, /^\s*Asegurado\s+Adicional\s+(\S.*)$/i); + const legalAddress = espectBlock(lines, /^\s*Ubicaci[óo]n\s+del\s+riesgo\s+(.+)$/i); + + // No ZIP field on this layout; it is the trailing run of the address + // ("…BAJA CALIFORNIA 22713"). Take the LAST five-digit run — a street + // number earlier in the line is never five digits, but anchoring on the + // first match would still be the fragile choice. + let zip: string | null = null; + if (legalAddress) { + const m = legalAddress.match(/(\d{5})(?!.*\d{5})/); + if (m) { + zip = m[1]; + notes.push(`código postal leído de la ubicación del riesgo (${m[1]})`); + } + } + + // ----- underwriting context --------------------------------------------- + // Not `ParsedPolicy` fields, but they are the difference between a coverage + // a reviewer can price and one they cannot, so they ride along in notes. + const personType = espectBlock(lines, /^\s*Tipo\s+Persona\s+Asegurada\s+(.+)$/i); + if (personType) notes.push(`tipo de persona asegurada: ${personType}`); + const building = espectBlock(lines, /^\s*Caracter[íi]sticas\s+del\s+Inmueble\s+(.+)$/i); + if (building) notes.push(`características del inmueble: ${building}`); + const water = firstMatch(text, [/^\s*(-?\s*\d+\s*mts\.?\s*cuerpo\s+agua\s+\S+)\s*$/im]); + if (water) notes.push(`distancia a cuerpo de agua: ${water.replace(/\s+/g, " ")}`); + const generalConditions = firstMatch(text, [/^\s*(W_\S+\.pdf)\s*$/im]); + if (generalConditions) notes.push(`condiciones generales: ${generalConditions}`); + + // ----- coverages --------------------------------------------------------- + const { coverages, currency } = parseEspecificacionCoverages(lines, notes); + + // These are absent by design on this document, not failures to read. Say so + // explicitly, or the reviewer reads four empty fields as a broken parse. + notes.push( + "la especificación PVL no trae vigencia, agente ni prima; esos datos están en la carátula de la póliza", + ); + + return { + provider: "GMX", + policyNumber, + insuredName, + additionalInsured, + agentName: null, + legalAddress, + zip, + policyFrom: null, + policyTo: null, + policyDate: null, + currency, + netPremium: null, + policyFee: null, + brokerFee: null, + total: null, + premiumPayment: null, + coverages, + notes, + }; +} + +/** + * Read a `Label Value` cell whose value wraps onto the following lines. + * + * `pdftotext -layout` renders the especificación's header cells with the + * value in a right-hand column, so a two-line risk location comes back as the + * label line plus a continuation line indented to the same column. A blank + * line always terminates the cell. + */ +function espectBlock(lines: string[], label: RegExp): string | null { + for (let i = 0; i < lines.length; i++) { + const m = lines[i].match(label); + if (!m?.[1]) continue; + const parts = [m[1]]; + for (let j = i + 1; j < lines.length && lines[j].trim(); j++) { + parts.push(lines[j].trim()); + } + const value = parts.join(" ").replace(/\s+/g, " ").trim(); + return value || null; + } + return null; +} + +/** + * Walk the especificación's coverages by anchoring on the limit label. + * + * For each `Límite … Responsabilidad:` the amount is either on the same line + * ("Bienes a la Intemperie"), on the next line (most), or split across two + * lines under `Edificio` / `Contenidos` sub-labels ("Remoción de escombros"). + * The risk name is the nearest heading ABOVE the label: a short line that is + * itself preceded by a blank one. That last condition is what distinguishes a + * heading from the short tail of a wrapped paragraph ("efectuados.", + * "Y CADA PÉRDIDA."), which is otherwise indistinguishable by length alone + * and would name a coverage after the last word of the prose above it. + * + * Returns the currency alongside, because on this layout there is no currency + * field — it is only ever printed next to the amounts, and the sublimits in + * the body prose are quoted in M.N. while the limits themselves are in USD. + * Only the limit amounts vote. + */ +function parseEspecificacionCoverages( + lines: string[], + notes: string[], +): { coverages: ParsedCoverage[]; currency: string | null } { + const coverages: ParsedCoverage[] = []; + const currencyVotes: Record = {}; + let lastHeading: string | null = null; + + const limitLines: number[] = []; + lines.forEach((line, i) => { + if (ESPEC_LIMIT_LABEL.test(line)) limitLines.push(i); + }); + + limitLines.forEach((i, n) => { + const nextLimit = limitLines[n + 1] ?? lines.length; + const heading = espectHeadingAbove(lines, i); + + // "Sublímites:" / "Límite adicional." head a block that qualifies the + // coverage above them rather than naming a new one, so they are read as + // a sublimit OF the previous risk instead of as a standalone coverage. + let risk = heading ?? "(sin nombre)"; + if (/^Subl[íi]mites?\b|^L[íi]mite\s+adicional/i.test(risk) && lastHeading) { + risk = `${lastHeading} — sublímite`; + } else if (heading) { + lastHeading = heading; + } + if (!heading) notes.push(`un límite no tiene encabezado identificable (línea ${i + 1})`); + + const deductible = espectDeductibleFor(lines, i, nextLimit); + const found = espectAmountsFor(lines, i); + + if (found.length === 0) { + notes.push(`no se pudo leer el monto de "${risk}"`); + coverages.push({ risk, insuredAmount: null, deductible, lossParticipation: null }); + return; + } + + for (const a of found) { + if (a.currency) currencyVotes[a.currency] = (currencyVotes[a.currency] ?? 0) + 1; + coverages.push({ + risk: a.sublabel ? `${risk} — ${a.sublabel}` : risk, + insuredAmount: a.amount, + deductible, + lossParticipation: null, + }); + } + }); + + // The per-section coverages that are stated inline rather than under a + // limit label. On this policy they are the two that matter most on the + // coast — earthquake (EXCLUIDO here) and hydrometeorological — and skipping + // them would leave the biggest exposure off the parsed row entirely. + coverages.push(...espectInlineSectionRisks(lines, currencyVotes, notes)); + + if (coverages.length === 0) notes.push("no se encontraron coberturas"); + + const currency = + Object.entries(currencyVotes).sort((a, b) => b[1] - a[1])[0]?.[0] ?? null; + if (currency) { + notes.push(`moneda tomada de los límites impresos (${currency})`); + } else { + notes.push("no se pudo determinar la moneda de las sumas aseguradas"); + } + + return { coverages, currency }; +} + +/** + * The nearest line above `at` that reads as a heading: short, not page + * furniture, not a structural label, and itself preceded by a blank line. + */ +function espectHeadingAbove(lines: string[], at: number): string | null { + for (let i = at - 1, walked = 0; i >= 0 && walked < 40; i--, walked++) { + if (!isEspecHeading(lines, i)) continue; + return lines[i].trim().replace(/\s*[:.]$/, ""); + } + return null; +} + +/** + * Does line `i` read as a coverage heading? + * + * Short, not page furniture, not a structural label — and preceded by a blank + * line, which is the condition that actually does the work: a heading sits in + * its own paragraph, and the short tail of a wrapped sentence ("efectuados.", + * "Y CADA PÉRDIDA.") does not. Length alone cannot tell those apart. + */ +function isEspecHeading(lines: string[], i: number): boolean { + const line = lines[i]?.trim() ?? ""; + if (!line) return false; + if (ESPEC_FURNITURE.test(line)) return false; + // The repeated policy-number line in the page header. + if (POLICY_NUMBER_SHAPE.test(line) && line.replace(/[\d\sOIlSBD-]/g, "") === "") return false; + if (ESPEC_STRUCTURAL.test(line)) return false; + if (line.length > 60) return false; + // An amount on its own line is a value, never a heading — the sublimit + // block prints one after a blank line, exactly where a heading would sit. + if (/^\$?\s*[\d,]+(?:\.\d{2})?\s*(USD|MXN|M\.?\s?N\.?|DLLS?)?$/i.test(line)) return false; + return i > 0 && !lines[i - 1].trim(); +} + +/** + * Amounts belonging to one limit label: on the label line itself, then on the + * following lines up to the blank that closes the block. A short non-amount + * line inside the block is a sub-label for the amount beneath it + * (`Edificio` / `Contenidos`). + */ +function espectAmountsFor( + lines: string[], + at: number, +): { amount: number | null; currency: string | null; sublabel: string | null }[] { + const out: { amount: number | null; currency: string | null; sublabel: string | null }[] = []; + + const push = (line: string, sublabel: string | null) => { + const m = line.match(ESPEC_AMOUNT); + if (!m) return false; + out.push({ amount: money(m[1]), currency: espectCurrency(m[2]), sublabel }); + return true; + }; + + const tail = lines[at].match(ESPEC_LIMIT_LABEL)?.[1]?.trim() ?? ""; + if (tail && push(tail, null)) return out; + + let sublabel: string | null = null; + let seenContent = false; + for (let i = at + 1; i < lines.length; i++) { + const line = lines[i].trim(); + // A blank line closes the block — but only once the block has started. + // The label and its amount are not always adjacent: the sublimit block on + // page 7 renders as "Límite de Responsabilidad:", a blank line, then + // "$2,000.00 USD", and breaking on the first blank drops that amount. + if (!line) { + if (seenContent) break; + if (i - at > 3) break; + continue; + } + if (ESPEC_FURNITURE.test(line)) continue; + if (push(line, sublabel)) { + sublabel = null; + seenContent = true; + continue; + } + seenContent = true; + // Not an amount: a sub-label if it is short enough to be one. + if (line.length <= 40 && !ESPEC_STRUCTURAL.test(line)) sublabel = line.replace(/\s*:$/, ""); + } + return out; +} + +/** + * The deductible printed inside this coverage's own block, i.e. between its + * limit and the next one. + * + * The value follows a bare `Deducible:` on the next line ("Sin deducible"). + * The length guard is load-bearing: the document also has a page-level + * `DEDUCIBLES:` heading followed by a paragraph explaining how deductibles + * interact with the inflation clause, and without the guard that paragraph + * becomes the deductible of whichever coverage precedes it. + */ +function espectDeductibleFor(lines: string[], from: number, to: number): string | null { + for (let i = from + 1; i < to && i < from + 30; i++) { + const line = lines[i].trim(); + // A heading ends this coverage's block. Not every block that follows a + // limit carries one of its own — the per-zone `Fenómenos + // hidrometeorológicos` deductible table has no limit label to stop at, so + // without this the coverage above it silently adopts its "1 POR CIENTO + // SOBRE SUMA ASEGURADA". + if (isEspecHeading(lines, i)) break; + const inline = line.match(/^Deducibles?\s*:\s*(\S.*)$/i); + if (inline) return inline[1].trim(); + if (!/^Deducibles?\s*:?\s*$/i.test(line)) continue; + for (let j = i + 1; j < to; j++) { + const value = lines[j].trim(); + if (!value) continue; + // Too long to be a deductible: this is the page-level `DEDUCIBLES:` + // heading and the paragraph explaining the inflation clause, not this + // coverage's value. Keep looking rather than concluding "none". + if (value.length <= 60) return value; + break; + } + } + return espectPercentDeductibleAbove(lines, from); +} + +/** + * Some coverages print their deductible and coinsurance as a sentence ABOVE + * the limit instead of under a `Deducible:` label below it — "Bienes a la + * Intemperie" reads + * + * 5 POR CIENTO SOBRE SUMA ASEGURADA, 20 PORCIENTO DE PARTICIPACIÓN A CARGO + * DEL ASEGURADO DE TODA Y CADA PÉRDIDA. + * + * wrapped across two lines. Left unread it looks like a coverage with no + * deductible at all, which is the one wrong answer worth avoiding here. + * + * Two bounds keep it from wandering into the body prose, and both are + * load-bearing — without them the fire section's "…o hasta el 10% de la suma + * asegurada de la sección de Edificio, lo que resulte menor" (a sublimit rule + * for the building, several paragraphs up) becomes the deductible of + * CONTENIDOS: + * + * - the walk stops at this coverage's own heading, so it can never read text + * belonging to the coverage above it, and + * - the line must OPEN with the percentage, which a prose sentence that + * merely contains one does not. + */ +function espectPercentDeductibleAbove(lines: string[], limitAt: number): string | null { + for (let i = limitAt - 1, walked = 0; i >= 0 && walked < 6; i--, walked++) { + const line = lines[i].trim(); + if (!line) continue; + // The heading of this coverage: stop, everything above belongs to another. + if (line.length <= 60 && !ESPEC_STRUCTURAL.test(line) && i > 0 && !lines[i - 1].trim()) { + return null; + } + if (!/^\d+\s*(POR\s*CIENTO|PORCIENTO|%)/i.test(line)) continue; + // Re-join the wrapped continuation lines beneath it. + const parts = [line]; + for (let j = i + 1; j < limitAt && lines[j].trim(); j++) parts.push(lines[j].trim()); + return parts.join(" ").replace(/\s+/g, " "); + } + return null; +} + +/** + * Coverages the especificación states inline inside the fire section instead + * of under a limit label: + * + * "2. Terremoto o erupción volcánica: Sección Edificio EXCLUIDO, Sección Contenidos EXCLUIDO" + * "3. Fenómenos hidrometeorológicos: Sección Edificio $200,000.00 USD, Sección Contenidos $20,000.00 USD" + * + * An exclusion is recorded in the risk label rather than as a zero amount: + * a coverage that is excluded and a coverage insured for $0 are the same + * number and very different facts, and `ParsedCoverage` has no field for the + * distinction. + */ +function espectInlineSectionRisks( + lines: string[], + currencyVotes: Record, + notes: string[], +): ParsedCoverage[] { + const out: ParsedCoverage[] = []; + const cat = espectCatastrophicTerms(lines, notes); + + for (const raw of lines) { + const m = raw.trim().match(/^\d+\.\s*([^:]{3,60}?)\s*:\s*(.*Secci[óo]n.+)$/i); + if (!m) continue; + const risk = m[1].trim(); + for (const part of m[2].split(/,\s*(?=Secci[óo]n)/i)) { + const section = part.match(/Secci[óo]n\s+(\S+)/i)?.[1] ?? null; + const label = section ? `${risk} — Sección ${section}` : risk; + // Deductible/coinsurance for these two live in their own block further + // down the document, keyed by section rather than printed here. + const terms = /hidrometeorol/i.test(risk) && section ? cat[section.toLowerCase()] : undefined; + + if (/EXCLUIDO|NO\s+CUBIERT/i.test(part)) { + out.push({ + risk: `${label}: EXCLUIDO`, + insuredAmount: null, + deductible: null, + lossParticipation: null, + }); + continue; + } + const amount = part.match(ESPEC_AMOUNT); + if (!amount) continue; + const currency = espectCurrency(amount[2]); + if (currency) currencyVotes[currency] = (currencyVotes[currency] ?? 0) + 1; + out.push({ + risk: label, + insuredAmount: money(amount[1]), + deductible: terms?.deductible ?? null, + lossParticipation: terms?.coinsurance ?? null, + }); + } + } + return out; +} + +/** + * The `Fenómenos hidrometeorológicos` block, which states the deductible and + * the coinsurance per section: + * + * Fenómenos hidrometeorológicos + * Zona: A2 + * Deducible + * Edificio: 1 POR CIENTO SOBRE SUMA ASEGURADA + * Coaseguro: + * Zona 1: (INTERIOR) Participación a cargo del asegurado del 10% … + * Zona 2: Participación a cargo del asegurado del 10% … + * + * The coinsurance is quoted per catastrophe zone, and the policy's own zone + * ("A2") is not one of the labels used in that list — so it is only read when + * every listed zone quotes the SAME percentage, which makes the mapping moot. + * When they differ, the reviewer is told to read the zone table by hand + * rather than being handed a percentage picked by guesswork. + */ +function espectCatastrophicTerms( + lines: string[], + notes: string[], +): Record { + const out: Record = {}; + + const start = lines.findIndex((l) => /^\s*Fen[óo]menos\s+hidrometeorol[óo]gicos\s*$/i.test(l)); + if (start < 0) return out; + let end = lines.length; + for (let i = start + 1; i < lines.length; i++) { + if (/^\s*(I+\.-\s*)?SECCI[ÓO]N\b/i.test(lines[i])) { + end = i; + break; + } + } + const block = lines.slice(start, end); + + const zone = firstMatch(block.join("\n"), [/^\s*Zona\s*:\s*(\S+)\s*$/im]); + if (zone) notes.push(`zona catastrófica declarada: ${zone}`); + + const percents = [ + ...new Set( + allMatches( + block.join("\n"), + /Participaci[óo]n\s+a\s+cargo\s+del\s+asegurado\s+del\s+(\d+%)/i, + ), + ), + ]; + let coinsurance: string | null = null; + if (percents.length === 1) { + coinsurance = percents[0]; + } else if (percents.length > 1) { + notes.push( + `el coaseguro de fenómenos hidrometeorológicos varía por zona (${percents.join(", ")}); revisarlo a mano`, + ); + } + + for (const line of block) { + const m = line.trim().match(/^(Edificio|Contenidos)\s*:\s*(\S.+)$/i); + if (!m) continue; + out[m[1].toLowerCase()] = { deductible: m[2].trim(), coinsurance }; + } + return out; +} + +/** `USD` / `M.N.` / `DLLS` as printed next to an amount → an ISO code. */ +function espectCurrency(raw: string | null | undefined): string | null { + if (!raw) return null; + const s = raw.replace(/[.\s]/g, "").toUpperCase(); + if (s === "USD" || s.startsWith("DL") || s.startsWith("DLL")) return "USD"; + if (s === "MN" || s === "MXN") return "MXN"; + return null; } \ No newline at end of file diff --git a/docs/POLICY_OCR.md b/docs/POLICY_OCR.md index 4a0d1d8..611e42b 100644 --- a/docs/POLICY_OCR.md +++ b/docs/POLICY_OCR.md @@ -84,8 +84,9 @@ feature's core assumption. Utility statements arrive **bundled, one customer per page** — so there, one page is one document and the parser runs per page. A policy PDF is the opposite: the GMX certificate is a 2-page document where page 1 carries the -contract header and page 2 carries the per-coverage table, and **both pages -describe the same policy**. So the pipeline concatenates every page's text +contract header and page 2 carries the per-coverage table (and the PVL +especificación runs to ten), and **every page describes the same policy**. So +the pipeline concatenates every page's text (`\n\n` between pages, which also keeps `ocrRawText` readable for debugging) and runs the parser and the matcher exactly **once per file**. @@ -133,6 +134,52 @@ customers do occur (one group policy bound by two related parties), and picking arbitrarily would silently book the wrong coverage against the wrong person. +## GMX ships two unrelated documents for the same policy + +The office downloads both from the same portal, and either can land in a +batch. They share only the brand and the policy number, so `parseGmx` is a +two-line dispatcher over two real parsers — both returning `provider: "GMX"`, +because the matcher keys on the policy number alone and must not care which +artifact was uploaded. + +| | **Caratula** (`…_Traduccion.pdf`) | **Especificación** (`…-CondicionesParticulares.pdf`) | +|---|---|---| +| Language | English (free translation) | Spanish | +| Shape | boxed header table + 4-column coverage table | 10 pages of prose, no tables at all | +| Header fields | Policy / Insured / Broker / Term / From / To / Currency | insured name, risk location, property description | +| Dates, broker, currency field | yes | **none printed** | +| Coverages | one row per risk | section heading + `Límite Máximo de Responsabilidad:` | +| Parser | `parseGmxCaratula` | `parseGmxEspecificacion` | + +Selected by `isEspecificacion` on the PVL page header +(`ESPECIFICACIÓN QUE SE ADHIERE`, `PVL Hogar`, `Nombre del asegurado`). + +Three things about the especificación are worth knowing before touching it: + +- **The policy number's group widths differ between the two.** The caratula + reads `007-037-07005947-0000-02` and the especificación + `07-037-07006957-00000-01` — 2 digits in the first group, 5 in the fourth. + The original parser pinned the widths, so it read one family and returned + null on the other. `POLICY_NUMBER_SHAPE` now matches the shape, and since + the especificación prints the number on all ten page headers, the ten + readings cross-check each other (disagreement is noted, not resolved — the + same rule the zona federal parser applies to its clave). +- **Coverages are found by anchoring on the limit label and walking backwards + for the heading.** There is no row shape to match. A heading is a short line + *preceded by a blank line* — that last condition is the whole trick, since + length alone cannot tell a heading from the wrapped tail of the paragraph + above it (`efectuados.`, `Y CADA PÉRDIDA.`), and without it coverages get + named after the last word of the preceding prose. +- **Vigencia, agente and prima are absent by design**, not unread. The parser + says so in a note, so a reviewer seeing four empty fields does not read it + as a broken parse. The caratula is the document that carries them. + +An excluded catastrophic risk is recorded as excluded **in the risk label** +(`Terremoto o erupción volcánica — Sección Edificio: EXCLUIDO`) with a null +amount, never as `0`: a coverage insured for zero and an excluded coverage are +the same number and very different facts, and `ParsedCoverage` has no field +for the distinction. + ## What the parser reads, and the field it cannot `ParsedPolicy` fields are all nullable on purpose: each carrier prints a @@ -210,13 +257,35 @@ feature is disabled. ## Tests -`apps/api/src/policy-ocr/parsers/policy-parser.spec.ts` — 8 cases, all -against verbatim text extracted from one real document, -`HC_Folio_000767_Traduccion.pdf`: provider detection from the wordmark and -from the footer URL, the header fields, every coverage row off the second -page, the deductible/loss-participation strings, the missing-premium note, -the broker line with the agent-number parens absent, and a page with no GMX -signal at all (which must yield no provider rather than a bad guess). +`apps/api/src/policy-ocr/parsers/policy-parser.spec.ts` — 24 cases against +verbatim text extracted from two real documents, indentation and blank lines +included (the column positions are what the parser reads, so a cleaned-up +fixture would test nothing). + +From `HC_Folio_000767_Traduccion.pdf` (caratula): provider detection from the +wordmark and from the footer URL, the header fields, every coverage row off +the second page, the deductible/loss-participation strings, the +missing-premium note, the broker line with the agent-number parens absent, +and a page with no GMX signal at all (which must yield no provider rather +than a bad guess). + +From `007_LGS-HGMX_07006957_01_0-CondicionesParticulares.pdf` +(especificación): the differently-grouped policy number, the risk location +read across its wrapped line, the empty `Asegurado Adicional` cell that must +not capture the next line, the absent-by-design fields, currency taken from +the USD limits rather than the M.N. sublimits in the body prose, a limit +split under `Edificio` / `Contenidos` sub-labels, a limit printed on the +label's own line, a deductible stated as a sentence *above* its limit, a +sublimit block whose amount sits after both a blank line and a page break, +the excluded earthquake coverage, and the hydrometeorological deductible and +coinsurance pulled from their own per-zone block. + +Four of those are regression tests for ways the parser can silently attach +the *wrong* value rather than none — a neighbouring coverage's prose read as +a deductible, the page-level `DEDUCIBLES:` paragraph read as one, a coverage +named after a wrapped prose tail, and one section's per-zone deductible +adopted by the coverage above it. Each was a real defect caught by running +the parser against the full ten-page document. ## Not built