diff --git a/apps/api/src/policy-ocr/parsers/policy-parser.spec.ts b/apps/api/src/policy-ocr/parsers/policy-parser.spec.ts index d7f2480..d6c22ea 100644 --- a/apps/api/src/policy-ocr/parsers/policy-parser.spec.ts +++ b/apps/api/src/policy-ocr/parsers/policy-parser.spec.ts @@ -859,6 +859,15 @@ describe("parsePolicy / ANA driver's policy (licencia)", () => { expect(parsePolicy(tripled).drivers).toHaveLength(1); }); + it("splits the phone off the name even without the printed column gap", () => { + // The phone shares the name cell, and the only thing marking it off is + // white space — which the OCR seam is free to collapse. Depending on the + // gap surviving is what put "PAMELA DENISE WAGONER Ph.3102001538" in the + // insured field, where it matched no customer. + const collapsed = page(ANA_LICENCIA.text.replace(/ {2,}/g, " ")); + expect(parsePolicy(collapsed).insuredName).toBe("PAMELA DENISE WAGONER"); + }); + it("drops the four empty driver slots", () => { // Slots 2-5 print an empty NAME and a bare "NONE" licence. expect(p.drivers.map((d) => d.fullName)).toEqual(["PAMELA DENISE WAGONER"]); diff --git a/apps/api/src/policy-ocr/parsers/policy-parser.ts b/apps/api/src/policy-ocr/parsers/policy-parser.ts index 72f1416..d0e024b 100644 --- a/apps/api/src/policy-ocr/parsers/policy-parser.ts +++ b/apps/api/src/policy-ocr/parsers/policy-parser.ts @@ -727,13 +727,22 @@ function parseGmxEspecificacion(page: OcrPage): ParsedPolicy { * value in a right-hand column, so a two-line risk location comes back as the * label line plus a continuation line indented to the same column. A blank * line always terminates the cell. + * + * The line budget is a guard, not the layout: the longest cell on this + * document wraps once. It exists because "walk until the cell ends" is only + * as good as the blank line it walks to, and when the OCR seam stopped + * emitting those, this returned the whole first page as the insured's name — + * a failure with no bad value to notice, just one enormous good one. */ +const ESPEC_MAX_WRAP = 3; + function espectBlock(lines: string[], label: RegExp): string | null { for (let i = 0; i < lines.length; i++) { const m = lines[i].match(label); if (!m?.[1]) continue; const parts = [m[1]]; - for (let j = i + 1; j < lines.length && lines[j].trim(); j++) { + const until = Math.min(lines.length, i + 1 + ESPEC_MAX_WRAP); + for (let j = i + 1; j < until && lines[j].trim(); j++) { parts.push(lines[j].trim()); } const value = parts.join(" ").replace(/\s+/g, " ").trim(); @@ -1889,7 +1898,12 @@ function parseAnaDrivers(lines: string[]): ParsedDriver[] { const raw = scope[at].match(/^\s*\d\.\s*NAME\s*:\s*(.*)$/i)?.[1]?.trim() ?? ""; if (!raw) return; - const phoneAt = raw.match(/\s{2,}Ph\.?\s*([\d()\s.-]{7,})\s*$/i); + // One space is enough to split on: the "Ph." marker plus seven digits at + // the end of the cell is not something a name does. Requiring the printed + // column gap made this depend on the reassembler keeping it, and when that + // collapsed the phone rode into `insuredName` ("PAMELA DENISE WAGONER + // Ph.3102001538") and no customer matched it. + const phoneAt = raw.match(/\s+Ph\.?\s*([\d()\s.-]{7,})\s*$/i); const fullName = (phoneAt ? raw.slice(0, phoneAt.index) : raw).replace(/\s+/g, " ").trim(); if (!fullName) return; diff --git a/apps/api/src/statements/ocr/tesseract.provider.spec.ts b/apps/api/src/statements/ocr/tesseract.provider.spec.ts index 71865ec..58875f9 100644 --- a/apps/api/src/statements/ocr/tesseract.provider.spec.ts +++ b/apps/api/src/statements/ocr/tesseract.provider.spec.ts @@ -7,8 +7,13 @@ import { parseBboxLayout } from "./tesseract.provider"; * that grouping is what left `PERIODO FACTURADO` with no value next to it and * every period field empty on a batch whose text was perfectly readable. */ +/** + * Boxes are sized from the text, at 6 units a character: the reassembler now + * reads the space BETWEEN two boxes, so a fixed width would put a fabricated + * gap after every short word and every row would come back column-padded. + */ function word(x: number, y: number, text: string): string { - return `${text}`; + return `${text}`; } function doc(...lines: string[]): string { @@ -26,23 +31,57 @@ describe("parseBboxLayout", () => { it("rejoins a label with the value printed beside it in another flow", () => { const [page] = parseBboxLayout( doc( - word(20, 100, "PERIODO") + word(45, 100, "FACTURADO:"), + word(20, 100, "PERIODO") + word(68, 100, "FACTURADO:"), word(300, 100.4, "20260630-20260630"), padding(), ), 1, ); expect(page).not.toBeNull(); - expect(page!.text).toContain("PERIODO FACTURADO: 20260630-20260630"); + expect(page!.text).toMatch(/PERIODO FACTURADO:\s+20260630-20260630/); }); it("keeps genuinely separate lines apart", () => { const [page] = parseBboxLayout( - doc(word(20, 100, "Cuenta:") + word(80, 100, "0900003463"), word(20, 130, "Nombre:"), padding()), + doc(word(20, 100, "Cuenta:") + word(68, 100, "0900003463"), word(20, 130, "Nombre:"), padding()), 1, ); - expect(page!.text.split("\n")).toContain("Cuenta: 0900003463"); - expect(page!.text.split("\n")).toContain("Nombre:"); + const lines = page!.text.split("\n").map((l) => l.trim()); + expect(lines).toContain("Cuenta: 0900003463"); + expect(lines).toContain("Nombre:"); + }); + + /** + * The layout is data. A borderless table separates its cells with nothing + * but white space, so the parsers read a run of spaces as a cell boundary + * (`INSURED\s{2,}`) and a column offset as a column (`SUM INSURED` vs + * `PREMIUM`). Both regressed to nothing when this collapsed every gap to a + * single space, and the fixtures — taken from `pdftotext -layout`, which + * prints the gaps — could not see it. + */ + it("preserves the gap between two cells of a borderless table", () => { + const [page] = parseBboxLayout( + doc(word(20, 100, "INSURED") + word(300, 100, "PAMELA") + word(340, 100, "WAGONER"), padding()), + 1, + ); + const line = page!.text.split("\n").find((l) => l.includes("INSURED"))!; + expect(line).toMatch(/INSURED\s{2,}PAMELA WAGONER/); + }); + + it("preserves the blank line between two blocks", () => { + const [page] = parseBboxLayout( + doc(word(20, 100, "Insured"), word(20, 112, "wraps"), word(20, 200, "Next"), padding()), + 1, + ); + const lines = page!.text.split("\n").map((l) => l.trim()); + // The wrapped continuation stays attached; the next block is cut off from + // it, which is what stops a "join until the cell ends" walk running away. + expect(lines.slice(lines.indexOf("Insured"), lines.indexOf("Next") + 1)).toEqual([ + "Insured", + "wraps", + "", + "Next", + ]); }); it("scales point coordinates into the render's pixel space", () => { diff --git a/apps/api/src/statements/ocr/tesseract.provider.ts b/apps/api/src/statements/ocr/tesseract.provider.ts index e25256c..685b111 100644 --- a/apps/api/src/statements/ocr/tesseract.provider.ts +++ b/apps/api/src/statements/ocr/tesseract.provider.ts @@ -254,6 +254,23 @@ export function parseBboxLayout(xhtml: string, scale: number): (OcrPage | null)[ * Rows are cut when a word's vertical centre leaves the band established by * the row's first word, which tolerates the sub-pixel baseline differences * between fonts on one line without merging two genuinely separate lines. + * + * Vertical WHITE SPACE is preserved as a blank line. Rows alone are not the + * whole layout: on a form, the blank between two blocks is what says where a + * cell's wrapped value stops, and dropping it leaves parsers that walk a + * block ("keep joining until the cell ends") running to the end of the page. + * That is not hypothetical — the GMX PVL especificación read its whole first + * page as the insured's name, because the fixtures were taken from + * `pdftotext -layout` (which prints the blanks) while the runtime fed it this + * function's output (which did not). + * + * Horizontal white space is preserved the same way, by padding each word out + * to its own column. The same fixture mismatch bit here: a run of spaces is + * the ONLY thing separating two cells of a borderless table, so ANA's + * `INSURED\s{2,}` label matches and its `SUM INSURED` / `PREMIUM` column + * split (taken from `head.search()` offsets) both need real offsets. Joining + * on one space put every driver's-policy premium in the sum-insured column + * and left the phone glued to the insured's name. */ function toVisualRows(words: OcrWord[]): string { const centre = (w: OcrWord) => w.top + w.height / 2; @@ -281,14 +298,85 @@ function toVisualRows(words: OcrWord[]): string { } if (current.length) rows.push(current); - return rows - .map((r) => - [...r] - .sort((a, b) => a.left - b.left) - .map((w) => w.text) - .join(" "), - ) - .join("\n"); + const charWidth = estimateCharWidth(words); + const out: string[] = []; + rows.forEach((r, i) => { + if (i > 0 && isBlankBetween(rows[i - 1], r)) out.push(""); + out.push(layoutRow(r, charWidth)); + }); + return out.join("\n"); +} + +/** + * One row rendered at its printed column offsets. + * + * Words that merely follow one another inside the same cell are separated by + * exactly one space, whatever the column arithmetic says: one `charWidth` for + * a page that mixes fonts leaves a rounding error on every word, and letting + * that accumulate sprinkles `\s{2,}` runs through ordinary prose — which is + * the very thing the parsers read as a cell boundary. Only a gap wide enough + * to be deliberate (more than one blank character) is rendered as one, and + * only there is the word re-anchored to its true column, so the offsets a + * column split depends on stay honest while values stay clean. + */ +function layoutRow(row: OcrWord[], charWidth: number): string { + let line = ""; + let right = 0; + + for (const w of [...row].sort((a, b) => a.left - b.left)) { + const col = Math.round(w.left / charWidth); + if (!line.length) { + line = " ".repeat(Math.max(0, col)); + } else if (w.left - right > charWidth * 1.5) { + line += " ".repeat(Math.max(2, col - line.length)); + } else { + line += " "; + } + line += w.text; + right = w.left + w.width; + } + + return line.trimEnd(); +} + +/** + * Width of one character, in the same units the word boxes use. + * + * The median of each word's own width-per-character: robust to the handful of + * oversized headings and to the wide-tracked letterhead, both of which would + * drag a mean. Only words of 3+ characters vote, since a one-character box is + * mostly side bearing. Falls back to a value derived from line height when a + * page has nothing long enough to measure. + */ +function estimateCharWidth(words: OcrWord[]): number { + const samples = words + .filter((w) => w.text.length >= 3 && w.width > 0) + .map((w) => w.width / w.text.length) + .sort((a, b) => a - b); + if (samples.length) return samples[Math.floor(samples.length / 2)]; + const heights = words.map((w) => w.height).filter((h) => h > 0); + return heights.length ? Math.max(...heights) / 2 : 1; +} + +/** + * Does the space between two consecutive rows read as an empty line? + * + * Measured against the taller of the two rows so a heading and its body text + * are judged on their own scale. On the real documents the two populations do + * not overlap: consecutive lines of one paragraph sit at 0.3–1.1 line heights + * apart, and anything the reader sees as blank-separated starts at 2.1. The + * threshold is placed in that empty middle, biased high — a missed blank only + * restores today's behaviour, while a spurious one would cut a wrapped value + * short. + */ +function isBlankBetween(prev: OcrWord[], row: OcrWord[]): boolean { + const bottom = Math.max(...prev.map((w) => w.top + w.height)); + const top = Math.min(...row.map((w) => w.top)); + const unit = Math.max( + ...prev.map((w) => w.height), + ...row.map((w) => w.height), + ); + return unit > 0 && top - bottom > unit * 1.6; } /**