feat(recibos): OCR capture for gas butano and municipal predial

Adds four parsers to the statement intake — GAS TIJUANA plus one per municipality, because Tijuana, Rosarito and Ensenada issue three completely different predial documents — and a text-layer fast path for the born-digital invoices the gas company sends. Measured against a new corpus of 14 documents / 29 pages: provider read on 29/29, amount on 26/29, and 21/29 auto-matched against the dev database (22/29 identified). The eight review cases are all legitimate. Five things the corpus forced: - Not every statement is a scan. The gas invoices are born-digital CFDIs whose text layer is exact; rasterising them only loses information (one sample turned `MEDIDOR: VM01014426` into `ar (LTR): 014420`). The new `OcrProvider.textPages` reads the embedded layer via `pdftotext -bbox-layout` — same poppler package as `pdftoppm`, so no new dependency — and OCR stays the fallback for real scans. Poppler's own `<line>` grouping follows text flow rather than the page, so words are regrouped by vertical position; without that, a two-column header leaves every label separated from the value printed beside it. - The clave catastral is not two letters and six digits. Position three is a letter in 15 of the 932 stored claves, and digitising the whole tail mapped a real `MMB01041` to a nonexistent `MM801041`. - Tijuana predial prints no clave at all. Its only identifier is an 8-digit municipal account carried in a 32-digit payment barcode, which the legacy database never held, so it goes in `meterNumber` alongside gas — `accountNumber` holds `DATMEX.predial`, which is not a per-property key and must not be overwritten. Those pages start cold and are taught by the first confirm. - On Rosarito and Ensenada the clave is the primary key, not a fallback: those receipts print nothing else, so a unique hit auto-matches. On a utility bill that merely happens to print one it stays a review hint. - A misread `$` is the dangerous failure. An Ensenada receipt for $2,203.00 OCR'd as `82,203.00`, which would post a charge 37x too large and look ordinary in the ledger. Predial amounts now require a literal `$` and a page that cannot produce one goes to review. The scoped match field is now one exported function rather than three copies of `kind === "GAS" ? ... : ...`, since the lookup, the blank-service fill and the confirm write-back have to agree or a reference gets learned into a column nothing searches. First tests in this package: 23 specs over the parsers and the text-layer reader, every fixture a verbatim OCR excerpt from a real receipt. Adds the jest config they need and a build tsconfig so they stay out of dist. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-01 12:52:20 -07:00
parent 216309190c
commit d6501f1d74
12 changed files with 940 additions and 58 deletions
@@ -7,7 +7,11 @@ import type { OcrPage, OcrWord } from "../ocr/ocr.provider";
 * like with like and never has to know about provider-specific formatting.
 */
 export interface ParsedStatement {
-  /** "CFE" | "CESPT" | "TELNOR", or null when no parser claimed the page. */
+  /**
+   * "CFE" | "CESPT" | "TELNOR" | "GAS TIJUANA" | "PREDIAL TIJUANA" |
+   * "PREDIAL ROSARITO" | "PREDIAL ENSENADA", or null when no parser claimed
+   * the page.
+   */
  provider: string | null;
  serviceKind: ServiceKind | null;
  accountRef: string | null;
@@ -90,6 +94,16 @@ function firstMatch(text: string, patterns: RegExp[]): string | null {
  return null;
 }

+/** Every capture of `pattern` across the page, in order. */
+function allMatches(text: string, pattern: RegExp): string[] {
+  const out: string[] = [];
+  const re = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`);
+  for (const m of text.matchAll(re)) {
+    if (m[1]) out.push(m[1].trim());
+  }
+  return out;
+}
+
 const MONTHS: Record<string, number> = {
  ENE: 0, FEB: 1, MAR: 2, ABR: 3, MAY: 4, JUN: 5,
  JUL: 6, AGO: 7, SEP: 8, OCT: 9, NOV: 10, DIC: 11,
@@ -104,15 +118,16 @@ export function parseDate(raw: string | null | undefined): Date | null {
  let m = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/);
  if (m) return utc(+m[3], +m[2] - 1, +m[1]);

-  // 22-JUL-2026  /  22 JUN 26
-  m = s.match(/^(\d{1,2})[-\s]([A-Z]{3})[A-Z]*[-\s](\d{2,4})$/);
+  // 22-JUL-2026  /  22 JUN 26  /  31/ENE/2026 (Tijuana predial)
+  m = s.match(/^(\d{1,2})[-\s/]([A-Z]{3})[A-Z]*[-\s/](\d{2,4})$/);
  if (m && MONTHS[m[2]] !== undefined) {
    const y = m[3].length === 2 ? 2000 + +m[3] : +m[3];
    return utc(y, MONTHS[m[2]], +m[1]);
  }

-  // 2026-07-22 (already normalised, e.g. decoded from a barcode)
-  m = s.match(/^(\d{4})-(\d{2})-(\d{2})$/);
+  // 2026-07-22 (already normalised, e.g. decoded from a barcode) and the
+  // 2026/08/08 the gas bill prints — same field order, different separator.
+  m = s.match(/^(\d{4})[-/](\d{2})[-/](\d{2})$/);
  if (m) return utc(+m[1], +m[2] - 1, +m[3]);

  return null;
@@ -174,9 +189,30 @@ const BRAND: [string, RegExp][] = [
  ["CFE", /comisi[oó]n federal de electricidad|CFE.?contigo|Suministrador de Servicios/i],
  ["CESPT", /CESPT|COMISI[OÓ]N ESTATAL DE SERVICIOS/i],
  ["TELNOR", /TELNOR|TELEFONOS DEL NOROESTE/i],
+  ["GAS TIJUANA", /COMPA[ÑN][IÍ]?A\s*DE\s*GAS\s*DE\s*TIJUANA|bajagas/i],
+  // The municipal RFCs are the single most reliable discriminator on a predial
+  // receipt: they are printed in a clean monospaced run on every layout, they
+  // never change, and they say which of the three city treasuries issued the
+  // page — which the wordmarks alone do not, since a Tijuana receipt also
+  // carries "PLAYAS DE TIJUANA" and a Rosarito one "TIJUANA ENSENADA".
+  ["PREDIAL TIJUANA", /AYUNTAMIENTO\s*DE\s*TIJUANA|ATB.?541201/i],
+  ["PREDIAL ROSARITO", /AYUNTAMIENTO\s*MUNICIPAL\s*DE\s*PLAYAS\s*DE\s*ROSARITO|AMP.?981201|rosarito\.gob/i],
+  ["PREDIAL ENSENADA", /MUNICIPIO\s*DE\s*ENSENADA|MEN.?540301/i],
 ];

+/**
+ * The predial rules come first because a Rosarito receipt prints "Clave
+ * Catastral" as a boxed label — the very string the CESPT structural rule
+ * looks for — so a page whose municipal header failed to OCR would otherwise
+ * be claimed as a water bill and matched against the wrong column entirely.
+ * "IMPUESTO PREDIAL" appears on all three municipal layouts and on none of the
+ * utility ones, so it is the safe first question to ask.
+ */
 const LAYOUT: [string, RegExp][] = [
+  ["PREDIAL TIJUANA", /IMPUESTO\s*PREDIAL[\s\S]*?(?:CERTIFICACION\s*DE\s*CAJA|PASEO\s*DEL\s*CENTENARIO|PAGA\s*TU\s*PREDIAL)/i],
+  ["PREDIAL ENSENADA", /(?:IMPUESTO\s*PREDIAL[\s\S]*?TRANSPENINSULAR)|(?:IMPRESION\s*MAQUINA\s*REGISTRADORA)/i],
+  ["PREDIAL ROSARITO", /IMPUESTO\s*PREDIAL/i],
+  ["GAS TIJUANA", /Orden\s*de\s*Facturaci[oó]n|FACTOR\s*DE\s*PRESI[OÓ]N|GAS\s*LP/i],
  ["CFE", /NO\.?\s*DE\s*SERVICIO|L[IÍ]MITE\s*DE\s*PAGO|PERIODO\s*FACTURADO/i],
  ["CESPT", /SALDO\s+CORRIENTE|CLAVE\s*CATASTRAL|No\.?\s*DE\s*CUENTA/i],
  ["TELNOR", /Mes\s*de\s*Facturaci[oó]n|Pagar\s*antes\s*de/i],
@@ -364,10 +400,315 @@ function parseTelnor(page: OcrPage): ParsedStatement {
  };
 }

+// --- GAS (Compañía de Gas de Tijuana / bajagas) ------------------------------
+
+/**
+ * These arrive as born-digital CFDI PDFs rather than scans, so the text layer
+ * (see `TesseractOcrProvider.textPages`) usually reads them exactly and the
+ * patterns below only have to be tolerant enough for the scanned case.
+ *
+ * The account number is printed three times — supply address, fiscal data, and
+ * the payment stub at the foot — which is a free cross-check: three readings
+ * that agree are near-certainly right, and any disagreement means one of them
+ * was misread and the page deserves a human glance.
+ *
+ * `Cuenta` is what the matcher compares, not `Contrato`. The migration
+ * recovered gas references out of `PropertyService.notes` into `meterNumber`
+ * and what sat there is the 9-digit account (`900003463`), printed here with a
+ * leading zero as `0900003463`.
+ */
+function parseGas(page: OcrPage): ParsedStatement {
+  const text = page.text;
+  const notes: string[] = [];
+
+  const seen = allMatches(text, /Cuenta\s*[:;.]?\s*([0-9OIlSBD]{6,12})/i).map((s) =>
+    toDigits(s).replace(/^0+/, ""),
+  );
+  const distinct = [...new Set(seen.filter(Boolean))];
+
+  let accountRef: string | null = null;
+  let crossChecked: boolean | null = null;
+  if (distinct.length === 1) {
+    accountRef = distinct[0];
+    if (seen.length > 1) crossChecked = true;
+  } else if (distinct.length > 1) {
+    // Majority wins — the stub and the two address blocks print the same
+    // number, so a single divergent reading is the misread one. It still goes
+    // to review: `crossChecked: false` is what keeps the batch from
+    // auto-matching a number one of three readings disagreed with.
+    const tally = new Map<string, number>();
+    for (const s of seen) tally.set(s, (tally.get(s) ?? 0) + 1);
+    accountRef = [...tally.entries()].sort((a, b) => b[1] - a[1])[0][0];
+    crossChecked = false;
+    notes.push(`el número de cuenta se leyó de ${distinct.length} formas distintas (${distinct.join(", ")})`);
+  }
+
+  const amount = money(
+    firstMatch(text, [
+      /TOTAL\s*A\s*PAGAR\s*[:;.]?\s*\$\s*([\d,]+\.\d{2})/i,
+      /Total\s*a\s*pagar\s*[:;.]?\s*\$\s*([\d,]+\.\d{2})/i,
+    ]),
+  );
+
+  // `20260630-20260630` — the range the bill was cut for. Both ends are the
+  // same reading date on every sample, so the period is reported as the ISO
+  // month rather than a range no ledger row would ever be searched by.
+  const facturado = firstMatch(text, [/PERIODO\s*FACTURADO\s*[:;.]?\s*(\d{8})\s*-\s*\d{8}/i]);
+  const period = facturado ? `${facturado.slice(0, 4)}-${facturado.slice(4, 6)}` : null;
+
+  return {
+    provider: "GAS TIJUANA",
+    serviceKind: "GAS",
+    accountRef: accountRef || null,
+    cadastralKey: null,
+    amount,
+    dueDate: parseDate(
+      firstMatch(text, [/Fecha\s*de\s*Vencimiento\s*[:;.]?\s*(\d{4}\s*\/\s*\d{2}\s*\/\s*\d{2})/i])?.replace(
+        /\s/g,
+        "",
+      ),
+    ),
+    period,
+    crossChecked,
+    notes,
+  };
+}
+
+// --- PREDIAL (municipal property tax) ---------------------------------------
+
+/**
+ * Normalise a printed clave catastral to the eight-character form
+ * `Property.cadastralKey` holds. The municipalities print it grouped
+ * (`KP-128-106`, `MM-B01-041`); the stored value drops the separators
+ * (`KP128106`, `MMB01041`).
+ *
+ * The shape is *not* two letters and six digits, which is the assumption that
+ * has to be resisted here. Across the 932 distinct claves on file, characters
+ * four through eight are digits without exception, but the third is a digit in
+ * 917 of them and one of `A`, `B`, `H`, `T` in the other fifteen. Running the
+ * whole tail through `toDigits` — which maps `B` to `8` — is what turned a real
+ * `MMB01041` into a nonexistent `MM801041`, so only positions four onward get
+ * that treatment and a letter in the third position is kept as printed.
+ *
+ * That leaves a genuine ambiguity at that one position: a `B` there might be a
+ * misread `8`, and 34 stored claves do carry an `8` there against six with a
+ * `B`. It is left as read rather than guessed, because a page that fails to
+ * match lands in the review queue where a human fixes it in seconds, while a
+ * page that matches the wrong property posts a charge to the wrong customer.
+ *
+ * The two-letter prefix is the other fragile part. Tesseract inserts a spurious
+ * `I` into letter pairs with some regularity — a real `MM-200-010` came back as
+ * `MIM-200-010` — so a run longer than two letters has its `I`/`L` dropped
+ * first, which recovers exactly that case. Anything still not two letters is
+ * truncated and flagged, because a wrong prefix silently matches the wrong
+ * property or, more often, nothing at all.
+ */
+export function normalizeCadastralKey(
+  raw: string,
+  notes: string[],
+): string | null {
+  const m = raw.match(/^([A-Za-z|]{2,5})[-\s]?([A-Za-z0-9|]{3})[-\s]?([0-9OIlSBD]{3})$/);
+  if (!m) return null;
+
+  let letters = m[1].toUpperCase().replace(/[^A-Z]/g, "");
+  if (letters.length > 2) {
+    const stripped = letters.replace(/[IL]/g, "");
+    if (stripped.length === 2) {
+      letters = stripped;
+    } else {
+      letters = letters.slice(0, 2);
+      notes.push(`la clave catastral se leyó como "${m[1]}"; se tomó "${letters}"`);
+    }
+  }
+  if (letters.length !== 2) return null;
+
+  const third = m[2][0].toUpperCase();
+  const tail =
+    (/[A-Z]/.test(third) ? third : toDigits(third)) +
+    toDigits(m[2].slice(1)) +
+    toDigits(m[3]);
+
+  return tail.length === 6 ? letters + tail : null;
+}
+
+/** The grouped clave as printed, anchored to its label when one survived OCR. */
+const GROUPED_CLAVE = "[A-Z|]{2,5}-[A-Z0-9OIlSBD]{3}-[0-9OIlSBD]{3}";
+
+function findCadastralKey(text: string, notes: string[]): string | null {
+  const labelled = firstMatch(text, [
+    new RegExp(`Clave\\s*Catastral\\s*[^A-Z0-9]{0,8}(${GROUPED_CLAVE})`, "i"),
+    new RegExp(`CLAVE\\s*[^A-Z0-9]{0,8}(${GROUPED_CLAVE})`, "i"),
+  ]);
+  if (labelled) return normalizeCadastralKey(labelled, notes);
+
+  // Ensenada's label ("CLAVE") lands inside a table header that OCRs into
+  // noise more often than not, so the bare grouped shape is accepted as a
+  // fallback. It is distinctive enough — two letters and two three-character
+  // groups joined by hyphens appears nowhere else on these pages.
+  const bare = firstMatch(text, [new RegExp(`\\b(${GROUPED_CLAVE})\\b`)]);
+  return bare ? normalizeCadastralKey(bare, notes) : null;
+}
+
+/**
+ * Tijuana: a "CERTIFICACIÓN DE CAJA" whose payment barcode is one 32-digit run
+ * of `account(8) + due date(DDMMYY) + amount(9) + folio(9)`, verified against
+ * all five sample pages. Municipal totals are whole pesos (the receipt itself
+ * carries a "Redondeo" line), so the barcode amount needs no decimal point.
+ *
+ * No clave catastral is printed anywhere on this layout — the 8-digit
+ * municipal account is the only identifier, and it is not a number the legacy
+ * database ever held. Until a reviewer confirms one, every Tijuana page lands
+ * in review; confirming teaches the matcher (see `learnAccountRefs`) so the
+ * same property matches itself next year.
+ */
+function parsePredialTijuana(page: OcrPage): ParsedStatement {
+  const text = page.text;
+  const notes: string[] = [];
+
+  const barcode = text.match(/(?<![0-9OIlSBD])([0-9OIlSBD]{32})(?![0-9OIlSBD])/);
+  const printedTotal = money(
+    firstMatch(text, [/TOTAL\s*A?\s*PAGAR\s*[:;.]?\s*\$?\s*([\d,]+\.?\d{0,2})/i]),
+  );
+
+  let accountRef: string | null = null;
+  let amount: number | null = printedTotal;
+  let dueDate: Date | null = null;
+  let crossChecked: boolean | null = null;
+
+  if (barcode) {
+    const run = toDigits(barcode[1]);
+    const d = run.slice(8, 14);
+    const fromBarcode = Number(run.slice(14, 23));
+    accountRef = run.slice(0, 8);
+    dueDate = parseDate(`20${d.slice(4, 6)}-${d.slice(2, 4)}-${d.slice(0, 2)}`);
+    notes.push("cuenta, importe y vencimiento leídos del código de barras");
+
+    if (printedTotal != null) {
+      // Guarding the money, not the account number: the printed total is the
+      // figure a human would key, so when the two disagree one of them is a
+      // misread peso amount and nothing should post unreviewed.
+      crossChecked = Math.abs(printedTotal - fromBarcode) < 0.5;
+      if (!crossChecked) {
+        notes.push(
+          `el total impreso (${printedTotal}) no coincide con el código de barras (${fromBarcode})`,
+        );
+      }
+    }
+    if (amount == null) amount = fromBarcode;
+  }
+
+  if (!dueDate) {
+    dueDate = parseDate(
+      firstMatch(text, [/FECHA\s*VENCE\s*[:;.]?\s*(\d{1,2}\/\w{3}\/\d{4})/i]),
+    );
+  }
+
+  return {
+    provider: "PREDIAL TIJUANA",
+    serviceKind: "PROPERTY_TAX",
+    accountRef: accountRef || null,
+    cadastralKey: null,
+    amount,
+    dueDate,
+    // The fiscal year, which is what the legacy ledger's `period` holds for
+    // predial ("2026" is its single most common value). It is read from the
+    // assessment table's year column, and failing that from the deadline: a
+    // predial bill for year N falls due on 31 January of year N.
+    period:
+      firstMatch(text, [/VALOR\s*FISCAL[\s\S]{0,160}?\b(20\d{2})\b/i]) ??
+      (dueDate ? String(dueDate.getUTCFullYear()) : null),
+    crossChecked,
+    notes,
+  };
+}
+
+/**
+ * Rosarito: a wide "CERTIFICACIÓN DE CAJA" keyed by clave catastral, with no
+ * account number of its own — the clave is the identifier, which is exactly
+ * what `Property.cadastralKey` holds, so these match on the first pass.
+ *
+ * The total is read with a negative lookbehind on "Sub": the receipt prints
+ * `Sub Total $5,409.39` (before the peso rounding) directly above
+ * `Total $5,409.00`, and taking the first "Total" on the page books 39 cents
+ * that the municipality did not charge. The lookbehind allows zero spaces
+ * because the label prints both ways — `Sub Total` on one sample and
+ * `SubTotal` on the next, and the tight one is what slipped past a fixed
+ * `Sub\s` and read $9,624.85 off a receipt for $9,625.00.
+ */
+function parsePredialRosarito(page: OcrPage): ParsedStatement {
+  const notes: string[] = [];
+  const text = page.text;
+
+  return {
+    provider: "PREDIAL ROSARITO",
+    serviceKind: "PROPERTY_TAX",
+    accountRef: null,
+    cadastralKey: findCadastralKey(text, notes),
+    amount: money(firstMatch(text, [/(?<!Sub\s{0,3})Total\s*[|:;.]?\s*\$\s*([\d,]+\.\d{2})/i])),
+    // "EXTEMPORANEO DESPUES DE: 31/01/2026" — the leading E is regularly eaten
+    // by the box rule printed over it, so the anchor starts at "XTEMPORANEO".
+    dueDate: parseDate(
+      firstMatch(text, [/XTEMPOR[AÁ]NEO\s*DESPU[EÉ]S\s*DE\s*[:;.]?\s*(\d{2}\/\d{2}\/\d{4})/i]),
+    ),
+    period: firstMatch(text, [/Periodo\s*por\s*Pagar\s*[:;.]?\s*(20\d{2})/i]),
+    crossChecked: null,
+    notes,
+  };
+}
+
+/**
+ * Ensenada: a dot-matrix "IMPRESION MAQUINA REGISTRADORA" statement, by some
+ * distance the worst-scanning of the three. Matching is by clave catastral.
+ *
+ * The amount is read positionally rather than by label, because the label does
+ * not survive: across five real pages the same word came back as `TOTAL:`,
+ * `TOTA LA A` and `orAL:`. What is stable is the row — the summary line that
+ * starts `TOTALES` carries the assessed figures across it and the amount
+ * actually paid last, at the right margin.
+ *
+ * That last figure must carry a literal `$`. On a real sample the paid total
+ * printed as `TOTAL: A $2,203.00` and OCR'd as `TOTAL: A 82,203.00` — the
+ * dollar sign read as an 8, a mistake that would post a $2,203 charge as
+ * $82,203 and look entirely ordinary in the ledger. Requiring the `$` costs
+ * that page its amount and sends it to review, which is the only acceptable
+ * failure here. The unprefixed figures earlier on the row are deliberately not
+ * a fallback: they are the tax assessed before the early-payment discount, not
+ * what was paid.
+ */
+function parsePredialEnsenada(page: OcrPage): ParsedStatement {
+  const notes: string[] = [];
+  const text = page.text;
+
+  const totalsRow = text.split("\n").find((l) => /TOTALES/i.test(l)) ?? "";
+  const figures = allMatches(totalsRow, /\$\s*(\d[\d,.\s]*\.\d{2})/);
+  const amount = figures.length ? money(figures[figures.length - 1]) : null;
+  if (amount == null) {
+    notes.push("no se pudo leer el importe con certeza; capturarlo a mano");
+  }
+
+  return {
+    provider: "PREDIAL ENSENADA",
+    serviceKind: "PROPERTY_TAX",
+    accountRef: null,
+    cadastralKey: findCadastralKey(text, notes),
+    amount,
+    // This layout prints no payment deadline at all — it is a receipt for a
+    // payment already made at the municipal window.
+    dueDate: null,
+    period: firstMatch(text, [/A[ÑN]O\s*[\s\S]{0,60}?\b(20\d{2})\b/i]),
+    crossChecked: null,
+    notes,
+  };
+}
+
 const PARSERS: Record<string, (page: OcrPage) => ParsedStatement> = {
  CFE: parseCfe,
  CESPT: parseCespt,
  TELNOR: parseTelnor,
+  "GAS TIJUANA": parseGas,
+  "PREDIAL TIJUANA": parsePredialTijuana,
+  "PREDIAL ROSARITO": parsePredialRosarito,
+  "PREDIAL ENSENADA": parsePredialEnsenada,
 };

 const EMPTY: ParsedStatement = {