diff --git a/apps/api/src/statements/parsers/statement-parser.spec.ts b/apps/api/src/statements/parsers/statement-parser.spec.ts index a9c042d..3955978 100644 --- a/apps/api/src/statements/parsers/statement-parser.spec.ts +++ b/apps/api/src/statements/parsers/statement-parser.spec.ts @@ -2,6 +2,7 @@ import type { OcrPage } from "../ocr/ocr.provider"; import { detectProvider, normalizeCadastralKey, + normalizeZofematKey, parseStatement, } from "./statement-parser"; @@ -192,3 +193,78 @@ describe("parseGas", () => { expect(parseStatement(gas("0900003463")).crossChecked).toBeNull(); }); }); + +describe("parseZonaFederal", () => { + /** + * The Tijuana zona federal receipt, trimmed to the rows the parser reads. + * Verbatim from page 7 of the August 2026 batch, including the two ways the + * heading OCR'd: the clave line is struck through by the office's own + * highlighter, which is what cost two of eight pages their concession clave. + */ + const zf = (clave: string, body = "") => + page( + "ESIZ { + // Tijuana bills predial and zona federal from the same treasury, so + // "Ayuntamiento de Tijuana" and ATB-541201 identify neither on their own. + expect(detectProvider("R.F.C. ATB-541201-BK2\nZona Federal Marítimo Terrestre")).toBe( + "ZONA FEDERAL TIJUANA", + ); + expect(parseStatement(zf("Clave: 14-D -014")).serviceKind).toBe("FEDERAL_ZONE"); + }); + + it("still recognises the layout when the heading itself did not survive OCR", () => { + // Real: page 1 came back as "Zona Ledera) Maritimo Terrestre". + expect( + detectProvider("Zona Ledera) Maritimo Terrestre\nClave EJ -012% Nombre: STEFAN"), + ).toBe("ZONA FEDERAL TIJUANA"); + }); + + it("reads the clave through the loose spacing the receipt prints", () => { + expect(parseStatement(zf("Clave: 14-D -014")).accountRef).toBe("14D014"); + expect(parseStatement(zf("Clave: 14-A-119")).accountRef).toBe("14A119"); + }); + + it("keeps the letter instead of digitising it", () => { + // toDigits maps D to 0 and B to 8; a real 14-D -014 must not become 140014. + expect(normalizeZofematKey("14-D -014")).toBe("14D014"); + expect(normalizeZofematKey("12-B -013")).toBe("12B013"); + }); + + it("takes the payable amount from the SubTotal row, rounded to whole pesos", () => { + // The municipality rounds and prints the difference as "Ajuste Ley Hacienda + // Mpal"; 2,872.79 is charged as $2,873.00. + expect(parseStatement(zf("Clave: 14-D -014")).amount).toBe(2873); + }); + + it("prefers the printed total and cross-checks it against the subtotal", () => { + const p = parseStatement(zf("Clave: 14-D -014", "Total a pagar $2,873.00")); + expect(p.amount).toBe(2873); + expect(p.crossChecked).toBe(true); + }); + + it("sends a printed total that contradicts the subtotal to review", () => { + const p = parseStatement(zf("Clave: 14-D -014", "Total a pagar $2,973.00")); + expect(p.crossChecked).toBe(false); + }); + + it("translates the printed bimester into the ledger's own vocabulary", () => { + expect(parseStatement(zf("Clave: 14-D -014")).period).toBe("MAR/APR"); + }); + + it("leaves the clave blank rather than guessing when the marker ate it", () => { + const p = parseStatement(zf("Clave EJ -012%")); + expect(p.accountRef).toBeNull(); + expect(p.notes.join(" ")).toContain("clave"); + }); +}); diff --git a/apps/api/src/statements/parsers/statement-parser.ts b/apps/api/src/statements/parsers/statement-parser.ts index 6f65bfb..7950e3a 100644 --- a/apps/api/src/statements/parsers/statement-parser.ts +++ b/apps/api/src/statements/parsers/statement-parser.ts @@ -9,8 +9,8 @@ import type { OcrPage, OcrWord } from "../ocr/ocr.provider"; export interface ParsedStatement { /** * "CFE" | "CESPT" | "TELNOR" | "GAS TIJUANA" | "PREDIAL TIJUANA" | - * "PREDIAL ROSARITO" | "PREDIAL ENSENADA", or null when no parser claimed - * the page. + * "PREDIAL ROSARITO" | "PREDIAL ENSENADA" | "ZONA FEDERAL TIJUANA", or null + * when no parser claimed the page. */ provider: string | null; serviceKind: ServiceKind | null; @@ -190,6 +190,15 @@ const BRAND: [string, RegExp][] = [ ["CESPT", /CESPT|COMISI[OÓ]N ESTATAL DE SERVICIOS/i], ["TELNOR", /TELNOR|TELEFONOS DEL NOROESTE/i], ["GAS TIJUANA", /COMPA[ÑN][IÍ]?A\s*DE\s*GAS\s*DE\s*TIJUANA|bajagas/i], + // Ahead of the predial rules on purpose. Tijuana's zona federal receipt is + // issued by the same treasury and carries the same header — "Ayuntamiento de + // Tijuana", the same address, the same `ATB-541201` RFC — so every predial + // discriminator matches it too, and whichever rule is asked first wins the + // page. What only the zona federal layout says is "Marítimo Terrestre", which + // survived OCR on all eight sample pages even where the heading above it came + // back as "Zona Ledera) Maritimo Terrestre" and the printed concession clave + // was lost under a highlighter mark. + ["ZONA FEDERAL TIJUANA", /ZOFEMAT|Mar[ií]timo\s*Terrestre|ocupaci[oó]n\s*de\s*Zona\s*Federal/i], // The municipal RFCs are the single most reliable discriminator on a predial // receipt: they are printed in a clean monospaced run on every layout, they // never change, and they say which of the three city treasuries issued the @@ -209,6 +218,11 @@ const BRAND: [string, RegExp][] = [ * utility ones, so it is the safe first question to ask. */ const LAYOUT: [string, RegExp][] = [ + // Same reasoning as the brand pass, one rule earlier: the concept line + // "Derechos de ocupación de Zona Federal Marítimo Terrestre" is printed on + // the stub of every zona federal page and on no other layout, and it read + // cleanly on 8 of 8 samples — including the two whose heading did not. + ["ZONA FEDERAL TIJUANA", /Derechos\s*de\s*ocupaci[oó]n/i], ["PREDIAL TIJUANA", /IMPUESTO\s*PREDIAL[\s\S]*?(?:CERTIFICACION\s*DE\s*CAJA|PASEO\s*DEL\s*CENTENARIO|PAGA\s*TU\s*PREDIAL)/i], ["PREDIAL ENSENADA", /(?:IMPUESTO\s*PREDIAL[\s\S]*?TRANSPENINSULAR)|(?:IMPRESION\s*MAQUINA\s*REGISTRADORA)/i], ["PREDIAL ROSARITO", /IMPUESTO\s*PREDIAL/i], @@ -701,6 +715,132 @@ function parsePredialEnsenada(page: OcrPage): ParsedStatement { }; } +// --- ZONA FEDERAL (ZOFEMAT, Tijuana) ---------------------------------------- + +/** + * Normalise the concession clave the zona federal receipt is keyed by. + * + * It is printed grouped and loosely spaced — `12-T -012`, `14-A-119`, + * `14-K -031` — and is a different shape from the cadastral key entirely: two + * digits, one letter, three digits. The letter is kept as printed rather than + * digitised, for the same reason `normalizeCadastralKey` keeps its third + * character: `toDigits` maps `B` to `8` and `D` to `0`, and a real `14-D -014` + * run through it becomes `140014`, which is not a clave at all. + * + * Stored without separators, because nothing on file holds this value yet (see + * `parseZonaFederal`) so the canonical form is ours to pick, and a bare run + * cannot be broken by the hyphen the scan renders as a dash, a minus or + * nothing. + */ +export function normalizeZofematKey(raw: string): string | null { + const m = raw.match(/^([0-9OIlSBD]{2})\s*-\s*([A-Za-z])\s*-?\s*([0-9OIlSBD]{3})$/); + if (!m) return null; + const zone = toDigits(m[1]); + const lot = toDigits(m[3]); + if (zone.length !== 2 || lot.length !== 3) return null; + return `${zone}${m[2].toUpperCase()}${lot}`; +} + +/** + * The bimester the receipt prints as `2026-2 / 2026-2`, rendered in the + * vocabulary the ledger already speaks. + * + * All 258 legacy FEDERAL ZONE transactions carry a period of `JAN/FEB`, + * `MAR/APR`, `MAY/JUN` or `NOV/DEC`, and their payment dates confirm the + * ordering — JAN/FEB was paid in March, MAR/APR in May, MAY/JUN in July, + * NOV/DEC in January, i.e. always the month after the bimester closes. The + * receipts agree: the two `2026-3` samples fall due 17/07/2026 with no + * surcharge, which is bimester three, May and June. Writing `2026-3` instead + * would leave the OCR-posted rows unsearchable alongside every hand-keyed one. + */ +const BIMESTERS = ["JAN/FEB", "MAR/APR", "MAY/JUN", "JUL/AUG", "SEP/OCT", "NOV/DEC"]; + +/** + * Tijuana's "Zona Federal Marítimo Terrestre" — the federal maritime-zone + * occupancy fee, billed by the municipality for beachfront lots. + * + * Nothing on file identifies these. `PropertyService.accountNumber` for + * FEDERAL_ZONE holds DATMEX.zfed, which is not a reference at all but an + * amount: its 77 values include `246.06`, `2369.09`, `22653.94` and a negative + * `-1679`, and the concession claves these receipts are keyed by appear nowhere + * in the database. So the clave goes to `meterNumber` (see `scopedRefField`), + * every page starts cold, and the first confirm teaches the match — the same + * arrangement Tijuana predial needed, for the same reason. + * + * The amount is taken from the SubTotal row rather than the "Total a pagar" + * box, which is printed on a grey fill and OCR'd on only 1 of 8 sample pages + * while the SubTotal row read on 8 of 8. The two differ by design: the + * municipality rounds to whole pesos and prints the difference on its own + * "Ajuste Ley Hacienda Mpal" line — `-$0.05` against a 591.05 subtotal, `$0.21` + * against 2,872.79 — so the payable figure is the rounded subtotal, and where + * the printed box did read, it agreed. + */ +function parseZonaFederal(page: OcrPage): ParsedStatement { + const text = page.text; + const notes: string[] = []; + + // Printed twice, once on the receipt and once on the stub below it, which is + // a free second reading: on one sample the heading was struck through by the + // office's own highlighter and only the stub survived. + const claves = [ + ...new Set( + allMatches(text, /Clave\s*[:;.]?\s*([0-9OIlSBD]{2}\s*-\s*[A-Za-z]\s*-?\s*[0-9OIlSBD]{3})/i) + .map(normalizeZofematKey) + .filter((k): k is string => k != null), + ), + ]; + + const accountRef = claves[0] ?? null; + let crossChecked: boolean | null = null; + + const subtotalRow = text.split("\n").find((l) => /SubTotal/i.test(l)) ?? ""; + const figures = allMatches(subtotalRow, /(\d[\d,]*\.\d{2})/); + // Impuesto, Actualización, Recargo, Multa, Importe — the payable one is last. + const importe = figures.length ? money(figures[figures.length - 1]) : null; + const rounded = importe != null ? Math.round(importe) : null; + const printed = money( + firstMatch(text, [/Total\s*a\s*pagar\s*[:;.]?\s*\$?\s*([\d,]+\.\d{2})/i]), + ); + + if (printed != null && rounded != null) { + crossChecked = Math.abs(printed - rounded) < 0.5; + if (!crossChecked) { + notes.push( + `el total impreso (${printed}) no coincide con el subtotal redondeado (${rounded})`, + ); + } + } else if (rounded != null) { + notes.push("importe tomado del subtotal, redondeado al peso"); + } else if (printed == null) { + notes.push("no se pudo leer el importe con certeza; capturarlo a mano"); + } + + // A clave read two different ways means one of the two readings is wrong and + // there is no third to break the tie, so the page goes to a human even if the + // money cross-checked. + if (claves.length > 1) { + crossChecked = false; + notes.push(`la clave se leyó de ${claves.length} formas distintas (${claves.join(", ")})`); + } + if (!accountRef) notes.push("no se pudo leer la clave de la concesión"); + + const bimester = text.match(/\b(20\d{2})\s*-\s*([1-6])\s*\/\s*20\d{2}\s*-\s*[1-6]/); + + return { + provider: "ZONA FEDERAL TIJUANA", + serviceKind: "FEDERAL_ZONE", + accountRef, + cadastralKey: null, + amount: printed ?? rounded, + dueDate: parseDate( + firstMatch(text, [/Vencimiento\s*[:;.]?\s*(\d{2}\/\d{2}\/\d{4})/i]), + ), + period: bimester ? BIMESTERS[+bimester[2] - 1] : null, + crossChecked, + notes, + }; +} + const PARSERS: Record ParsedStatement> = { CFE: parseCfe, CESPT: parseCespt, @@ -709,6 +849,7 @@ const PARSERS: Record ParsedStatement> = { "PREDIAL TIJUANA": parsePredialTijuana, "PREDIAL ROSARITO": parsePredialRosarito, "PREDIAL ENSENADA": parsePredialEnsenada, + "ZONA FEDERAL TIJUANA": parseZonaFederal, }; const EMPTY: ParsedStatement = { diff --git a/apps/api/src/statements/statement-matcher.service.ts b/apps/api/src/statements/statement-matcher.service.ts index 6d86d7f..d46566a 100644 --- a/apps/api/src/statements/statement-matcher.service.ts +++ b/apps/api/src/statements/statement-matcher.service.ts @@ -40,15 +40,22 @@ export interface MatchResult { * When they disagree, a reference gets learned into a column nothing searches, * and the same page returns to the review queue every month forever. * - * `meterNumber` is doing double duty for the two kinds whose printed reference - * DATMEX never held in `accountNumber`: - * - GAS, where the number lived in free-text notes, and + * `meterNumber` is doing double duty for the three kinds whose printed + * reference DATMEX never held in `accountNumber`: + * - GAS, where the number lived in free-text notes, * - PROPERTY_TAX, where `accountNumber` holds DATMEX.predial — a 3-4 digit * office file number that is neither unique nor printed on any statement. * The Tijuana municipal receipt prints an 8-digit account and no clave * catastral at all, so it needs a column of its own; overwriting the legacy * predial numbers to make room would destroy the only link back to the - * original records. + * original records, and + * - FEDERAL_ZONE, where `accountNumber` holds DATMEX.zfed, which is not a + * reference of any kind but a peso amount: 3 of its 77 values carry cents + * (`246.06`, `2369.09`, `22653.94`) and one is negative. Searching it for + * the concession clave the receipt prints would never hit, and — worse — + * because every row already has a value, the `[field]: null` guards in + * `learnAccountRefs` and the blank-service fill would never fire either, so + * the same page would return to the review queue every bimester forever. */ export function scopedRefField( kind: ServiceKind, @@ -57,11 +64,11 @@ export function scopedRefField( case "ELECTRIC": // CFE "NO. DE SERVICIO" -> DATMEX.rpu case "WATER": // CESPT "Cuenta" / "No. DE CUENTA" -> DATMEX.agua case "TELEPHONE": // Telnor "Teléfono" (LADA stripped) -> DATMEX.telefono - case "FEDERAL_ZONE": case "CABLE": return "accountNumber"; case "GAS": // bajagas "Cuenta" -> recovered from notes into meterNumber case "PROPERTY_TAX": // Tijuana's 8-digit municipal account + case "FEDERAL_ZONE": // ZOFEMAT concession clave, e.g. `12T012` return "meterNumber"; default: return null; diff --git a/apps/web/src/components/StatementIntake.tsx b/apps/web/src/components/StatementIntake.tsx index 429c5ad..bc5ab75 100644 --- a/apps/web/src/components/StatementIntake.tsx +++ b/apps/web/src/components/StatementIntake.tsx @@ -34,9 +34,10 @@ const SUPPORTED: ServiceKind[] = [ "TELEPHONE", "GAS", "PROPERTY_TAX", + "FEDERAL_ZONE", ]; /** Uploadable, but every page will land in review until a parser learns it. */ -const OTHER_KINDS: ServiceKind[] = ["FEDERAL_ZONE", "CABLE"]; +const OTHER_KINDS: ServiceKind[] = ["CABLE"]; const STATUS_LABEL: Record = { UPLOADED: "Recibido", diff --git a/docs/RECEIPT_CAPTURE_SPEC.md b/docs/RECEIPT_CAPTURE_SPEC.md index dedd548..f8bc1b3 100644 --- a/docs/RECEIPT_CAPTURE_SPEC.md +++ b/docs/RECEIPT_CAPTURE_SPEC.md @@ -256,6 +256,54 @@ single-movement form. > `statement-parser.spec.ts` and `tesseract.provider.spec.ts`; every fixture in > them is a verbatim OCR excerpt from a real receipt. +> **EXTENDED — zona federal, 2026-08-01.** A third corpus (one document, 8 +> pages of Tijuana "Zona Federal Marítimo Terrestre" receipts — the federal +> maritime-zone occupancy fee billed on beachfront lots) added the +> `ZONA FEDERAL TIJUANA` parser. Provider read on 8/8, amount on 8/8 (all +> eight verified against the paper), concession clave on 6/8, period on 8/8, +> payment deadline on 2/8. Nothing auto-matched, and nothing could have — see +> point 2. +> +> **Four things this corpus proved:** +> +> 1. **Tijuana bills predial and zona federal from the same treasury.** Same +> "Ayuntamiento de Tijuana" header, same Paseo del Centenario address, same +> `ATB-541201` RFC — every discriminator the predial parser uses matches a +> zona federal page too, so whichever rule is asked first wins. The words +> only this layout prints are `Marítimo Terrestre`, so its brand rule is +> asked ahead of all three predial ones. +> 2. **`FEDERAL_ZONE.accountNumber` is an amount, not a reference.** It holds +> `DATMEX.zfed`, whose 77 values include `246.06`, `2369.09`, `22653.94` and +> a negative `-1679`; the concession claves the receipts are keyed by +> (`12-T -012`, `14-D -014`) appear nowhere in the database. Matching on that +> column could never hit — and because every row already has a value, the +> `[field]: null` guards on learning and on the blank-service fill would +> never fire either, so every page would return to review every bimester +> forever. The clave moves to `meterNumber`, joining gas and Tijuana predial, +> and the first confirm teaches the match. This is the same trap as +> `policies.total` and `PROPERTY_TAX.accountNumber`: a legacy column whose +> name promises an identifier and whose contents are something else. +> 3. **The payable figure is not the printed subtotal.** The municipality rounds +> to whole pesos and prints the difference as its own `Ajuste Ley Hacienda +> Mpal` line — `-$0.05` against a 591.05 subtotal, `$0.21` against 2,872.79. +> The "Total a pagar" box that carries the rounded figure sits on a grey fill +> and OCR'd on 1 of 8 pages; the SubTotal row read on 8 of 8. So the amount +> is the rounded subtotal, cross-checked against the printed box wherever it +> survives (it agreed). +> 4. **The office's own highlighter is an OCR failure mode.** Both pages that +> lost their clave lost it to a marker stroke drawn across the `Clave:` line +> — not to scan quality, which was otherwise fine. The clave is printed twice +> (receipt and stub), which rescued a third page whose heading was struck +> but whose stub was not; where both copies are struck, the page reports no +> clave and goes to review rather than guessing. +> +> **Not attempted:** deriving the payment deadline from the bimester. It is the +> 17th of the month after the bimester closes on a current bill, but four of +> these eight are late — they carry a $1,000 `Multa` — and print a +> recalculated deadline a month out. A derived date would be wrong on exactly +> the pages a human most wants to look at, so an unreadable deadline stays +> null. + ### Motivation (from the meeting) Each utility company (CFE, water, phone, gas...) sends 300+ individual