feat(statements): OCR intake for scanned utility bills

Staff key 300+ utility statements per company per month by hand. This adds the ingest -> split -> OCR -> match -> review pipeline that proposes customer and amount per page instead (RECEIPT_CAPTURE_SPEC §2), posting through the existing BillingService.createBatch seam with source=OCR and a per-document captureRef so machine and hand capture share one write path and audit trail. Everything was designed against 10 real scanned statements (46 pages of CFE, CESPT and Telnor bills) rather than from the sample-free spec. The scans have no text layer at all — they are camera images — so OCR is mandatory, and they arrive bundled one customer per page. Measured on those pages the parser identifies the provider 46/46 and reads an account reference 43/46; against the dev database that is 39/46 (85%) exact auto-match, 40/46 identified, with the rest genuine review cases. That closes the OCR-provider question in favour of self-hosted Tesseract: it clears the bar for a queue where a human confirms every row, and OcrProvider keeps a managed API a one-line swap. The samples corrected three things the spec had wrong or unknown: - Clave catastral is NOT predial. DATMEX.clave (934 rows) is what CESPT and predial bills print; DATMEX.predial, which PROPERTY_TAX.accountNumber holds, has 663 distinct values across 1135 rows and appears on no statement. The clave now lives on Property.cadastralKey as the matcher's secondary key; predial is left untouched. This had been blocking predial matching. - Gas was recoverable: 160 of 334 DATMEX.gas values are real account numbers (the rest are ESTACIONARIO/CILINDRO descriptors), now in GAS.meterNumber. - Phone is one billed line per property (534/18/1 across phone1/2/3), so the new TELEPHONE ServiceKind backfills from phone1 only, not three rows. Matching is scoped to one column per service kind and never reads the customer name — a CESPT receipt prints ARNAIZ ROSAS ELSA AURORA for an account this office holds under CATT, RANDY, because the printed name is the registrant, not the current owner. Where a provider prints a payment barcode it beats the printed label (one CFE label OCR'd a digit too many while its barcode was correct) and the two cross-check, with disagreement forcing review. Confirming a document whose service had no reference writes it back, so gas and any other cold start is a one-time cost rather than a permanent queue. Verified end to end against the live dev API and MinIO: real scans uploaded over HTTP, matched, confirmed against a check, and the resulting rows checked in MySQL (negative amounts, captureSource=OCR, concept derived from the batch kind, captureRef linking back to each page). Re-confirming a posted batch is refused. Test data was removed afterwards. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-01 00:42:35 -07:00
parent 121952fdc1
commit 4d5008b545
26 changed files with 3077 additions and 19 deletions
@@ -0,0 +1,390 @@
+import type { ServiceKind } from "@jorgecuadros/database";
+import type { OcrPage, OcrWord } from "../ocr/ocr.provider";
+
+/**
+ * What one parsed statement page yields. `accountRef` is already normalised to
+ * the form the migrated `PropertyService` columns hold, so the matcher compares
+ * like with like and never has to know about provider-specific formatting.
+ */
+export interface ParsedStatement {
+  /** "CFE" | "CESPT" | "TELNOR", or null when no parser claimed the page. */
+  provider: string | null;
+  serviceKind: ServiceKind | null;
+  accountRef: string | null;
+  /** Clave catastral, when printed — a second key to match on. */
+  cadastralKey: string | null;
+  amount: number | null;
+  dueDate: Date | null;
+  period: string | null;
+  /**
+   * Independent corroboration of `accountRef`. CFE and Telnor both print a
+   * payment barcode that repeats the account number (and the amount), so when
+   * the barcode and the label agree the extraction is near-certainly right;
+   * when they disagree, or only one is present, the page is worth a human
+   * glance. Null when the layout has no second source.
+   */
+  crossChecked: boolean | null;
+  /** Human-readable trail of what was read, surfaced in the review queue. */
+  notes: string[];
+}
+
+// --- shared helpers ---------------------------------------------------------
+
+/**
+ * Tesseract confuses these glyphs inside numeric runs with some regularity —
+ * a real clave catastral `KB078025` came back as `KBO78025`. Applied ONLY to
+ * fields known to be digits, never to free text, where it would corrupt words.
+ */
+const DIGIT_CONFUSIONS: Record<string, string> = {
+  O: "0",
+  o: "0",
+  D: "0",
+  I: "1",
+  l: "1",
+  "|": "1",
+  S: "5",
+  B: "8",
+};
+
+export function toDigits(s: string | null | undefined): string {
+  if (!s) return "";
+  return s
+    .split("")
+    .map((c) => DIGIT_CONFUSIONS[c] ?? c)
+    .join("")
+    .replace(/\D/g, "");
+}
+
+/**
+ * Parse a printed amount, treating `,` and `.` by position rather than by
+ * assumption. A real Telnor bill OCR'd as "$ 649,00" — blindly stripping commas
+ * as thousands separators turned $649.00 into $64,900, a hundredfold error that
+ * would post silently. Two trailing digits after a single separator are always
+ * cents here; a separator followed by three digits is a thousands group.
+ */
+function money(s: string | null | undefined): number | null {
+  if (!s) return null;
+  const cleaned = s.replace(/[\s$]/g, "");
+
+  // 1.234,56 or 1,234.56 — grouped thousands plus optional cents.
+  let m = cleaned.match(/^(\d{1,3}(?:[.,]\d{3})+)([.,]\d{1,2})?$/);
+  if (m) {
+    const whole = m[1].replace(/[.,]/g, "");
+    const cents = m[2] ? m[2].slice(1) : "";
+    return Number(cents ? `${whole}.${cents.padEnd(2, "0")}` : whole);
+  }
+
+  // 649,00 / 649.00 — a single separator with exactly two digits after it.
+  m = cleaned.match(/^(\d+)[.,](\d{2})$/);
+  if (m) return Number(`${m[1]}.${m[2]}`);
+
+  const n = Number(cleaned.replace(/[,.]/g, ""));
+  return Number.isFinite(n) ? n : null;
+}
+
+function firstMatch(text: string, patterns: RegExp[]): string | null {
+  for (const p of patterns) {
+    const m = text.match(p);
+    if (m?.[1]) return m[1].trim();
+  }
+  return null;
+}
+
+const MONTHS: Record<string, number> = {
+  ENE: 0, FEB: 1, MAR: 2, ABR: 3, MAY: 4, JUN: 5,
+  JUL: 6, AGO: 7, SEP: 8, OCT: 9, NOV: 10, DIC: 11,
+};
+
+/** Parses the three date shapes these statements actually print. */
+export function parseDate(raw: string | null | undefined): Date | null {
+  if (!raw) return null;
+  const s = raw.trim().toUpperCase();
+
+  // 16/07/2026
+  let m = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/);
+  if (m) return utc(+m[3], +m[2] - 1, +m[1]);
+
+  // 22-JUL-2026  /  22 JUN 26
+  m = s.match(/^(\d{1,2})[-\s]([A-Z]{3})[A-Z]*[-\s](\d{2,4})$/);
+  if (m && MONTHS[m[2]] !== undefined) {
+    const y = m[3].length === 2 ? 2000 + +m[3] : +m[3];
+    return utc(y, MONTHS[m[2]], +m[1]);
+  }
+
+  // 2026-07-22 (already normalised, e.g. decoded from a barcode)
+  m = s.match(/^(\d{4})-(\d{2})-(\d{2})$/);
+  if (m) return utc(+m[1], +m[2] - 1, +m[3]);
+
+  return null;
+}
+
+function utc(y: number, mo: number, d: number): Date | null {
+  const dt = new Date(Date.UTC(y, mo, d));
+  return Number.isNaN(dt.getTime()) ? null : dt;
+}
+
+/**
+ * Read the value printed *underneath* a column header.
+ *
+ * The CESPT "RECIBO" is a table: `No. DE CUENTA` is a header cell and its value
+ * sits in the row below it, so no amount of label-adjacent regex on line text
+ * can associate the two. This walks the word boxes instead — find the header
+ * word, then take the nearest word below it whose horizontal centre falls
+ * within the column.
+ */
+export function valueUnder(
+  page: OcrPage,
+  header: RegExp,
+  opts: { maxDy?: number; tolerance?: number; match?: RegExp } = {},
+): string | null {
+  const { maxDy = 300, tolerance = 200, match } = opts;
+  const centre = (w: OcrWord) => ({
+    x: w.left + w.width / 2,
+    y: w.top + w.height / 2,
+  });
+
+  for (const h of page.words.filter((w) => header.test(w.text))) {
+    const hc = centre(h);
+    const below = page.words
+      .filter((w) => {
+        const c = centre(w);
+        return c.y > hc.y && c.y <= hc.y + maxDy && Math.abs(c.x - hc.x) <= tolerance;
+      })
+      .sort((a, b) => centre(a).y - centre(b).y);
+
+    for (const w of below) {
+      if (!match || match.test(w.text)) return w.text;
+    }
+  }
+  return null;
+}
+
+// --- provider detection -----------------------------------------------------
+
+/**
+ * Brand wordmarks first, page structure only as a fallback — and the two passes
+ * must not be interleaved. Scanned logos OCR badly (one CESPT header came back
+ * as "E BAJA ES PAGO / EALIFORNIA", with neither "CESPT" nor "COMISIÓN ESTATAL"
+ * readable), so the structural pass is what rescues those pages. But a Telnor
+ * bill contains the words "Pagar antes de", which a CFE structural rule
+ * evaluated first will happily claim — running all brand checks before any
+ * structural check is what keeps that from happening.
+ */
+const BRAND: [string, RegExp][] = [
+  ["CFE", /comisi[oó]n federal de electricidad|CFE.?contigo|Suministrador de Servicios/i],
+  ["CESPT", /CESPT|COMISI[OÓ]N ESTATAL DE SERVICIOS/i],
+  ["TELNOR", /TELNOR|TELEFONOS DEL NOROESTE/i],
+];
+
+const LAYOUT: [string, RegExp][] = [
+  ["CFE", /NO\.?\s*DE\s*SERVICIO|L[IÍ]MITE\s*DE\s*PAGO|PERIODO\s*FACTURADO/i],
+  ["CESPT", /SALDO\s+CORRIENTE|CLAVE\s*CATASTRAL|No\.?\s*DE\s*CUENTA/i],
+  ["TELNOR", /Mes\s*de\s*Facturaci[oó]n|Pagar\s*antes\s*de/i],
+];
+
+export function detectProvider(text: string): string | null {
+  for (const group of [BRAND, LAYOUT]) {
+    for (const [name, pattern] of group) {
+      if (pattern.test(text)) return name;
+    }
+  }
+  return null;
+}
+
+// --- CFE (electric) ---------------------------------------------------------
+
+function parseCfe(page: OcrPage): ParsedStatement {
+  const text = page.text;
+  const notes: string[] = [];
+
+  // The payment barcode line repeats the service number, the due date (YYMMDD)
+  // and the amount in one fixed-width run, and reads far more reliably than the
+  // label: on one sample the label came back as "0059603001917" (a digit too
+  // many) while its barcode gave the correct "005960300191". So the barcode
+  // wins, and the label becomes the cross-check rather than the source.
+  const barcode = text.match(/\b01\s+([0-9OIlSBD]{12})\s+([0-9OIlSBD]{6})\s+([0-9OIlSBD]{9})\b/);
+  const label = firstMatch(text, [/NO\.?\s*DE\s*SERVICIO\s*[:;.]?\s*([0-9OIlSBD]{10,14})/i]);
+
+  let accountRef: string | null = null;
+  let amount: number | null = null;
+  let dueDate: Date | null = null;
+  let crossChecked: boolean | null = null;
+
+  if (barcode) {
+    // Leading zeros are print padding: DATMEX.rpu holds the bare 10 digits.
+    accountRef = toDigits(barcode[1]).replace(/^0+/, "");
+    amount = Number(toDigits(barcode[3]));
+    const d = toDigits(barcode[2]);
+    dueDate = parseDate(`20${d.slice(0, 2)}-${d.slice(2, 4)}-${d.slice(4, 6)}`);
+    notes.push("importe y vencimiento leídos del código de barras");
+    if (label) {
+      crossChecked = toDigits(label).replace(/^0+/, "") === accountRef;
+      if (!crossChecked) {
+        notes.push(
+          `el número impreso (${toDigits(label).replace(/^0+/, "")}) no coincide con el código de barras`,
+        );
+      }
+    }
+  } else if (label) {
+    accountRef = toDigits(label).replace(/^0+/, "");
+    notes.push("sin código de barras legible; número tomado de la etiqueta");
+  }
+
+  if (amount == null) {
+    amount = money(firstMatch(text, [/TOTAL\s*A\s*PAGAR\s*[:;.]?\s*\$?\s*([\d,]+\.?\d*)/i]));
+  }
+  if (!dueDate) {
+    dueDate = parseDate(
+      firstMatch(text, [/L[IÍ]MITE\s*DE\s*PAGO\s*[:;.]?\s*(\d{1,2}\s+\w{3}\s+\d{2,4})/i]),
+    );
+  }
+
+  return {
+    provider: "CFE",
+    serviceKind: "ELECTRIC",
+    accountRef: accountRef || null,
+    cadastralKey: null,
+    amount,
+    dueDate,
+    period: firstMatch(text, [
+      /PERIODO\s*FACTURADO\s*[:;.]?\s*(\d{1,2}\s+\w{3}\s+\d{2}\s*-\s*\d{1,2}\s+\w{3}\s+\d{2})/i,
+    ]),
+    crossChecked,
+    notes,
+  };
+}
+
+// --- CESPT (water) ----------------------------------------------------------
+
+/**
+ * Two different layouts arrive under the same brand:
+ *   - the line-oriented "COMPROBANTE DE PAGO" (`Cuenta : 7604192`), and
+ *   - the tabular "RECIBO", where `No. DE CUENTA` is a column header.
+ * Line patterns are tried first; anything they miss falls through to the
+ * geometric read, which is what the tabular layout needs.
+ */
+function parseCespt(page: OcrPage): ParsedStatement {
+  const text = page.text;
+  const notes: string[] = [];
+
+  let account = firstMatch(text, [/Cuenta\s*[:;.]?\s*([0-9OIlSBD]{5,9})/i]);
+  if (!account) {
+    account = valueUnder(page, /^CUENTA$/i, { match: /^[0-9OIlSBD]{5,9}$/ });
+    if (account) notes.push("número de cuenta leído de la columna del recibo");
+  }
+
+  let clave = firstMatch(text, [/Cve\.?\s*Cat\.?\s*[:;.]?\s*([A-Z]{2}\s?[0-9OIlSBD]{6})/i]);
+  if (!clave) {
+    clave = valueUnder(page, /^CATASTRAL$/i, { match: /^[A-Z]{2}[0-9OIlSBD]{6}$/i });
+    if (clave) notes.push("clave catastral leída de la columna del recibo");
+  }
+
+  let due = firstMatch(text, [/Fecha\s*Venc\s*[:;.]?\s*(\d{2}\/\d{2}\/\d{4})/i]);
+  if (!due) due = valueUnder(page, /^VENCIMIENTO$/i, { match: /^\d{2}\/\d{2}\/\d{4}$/ });
+
+  const amount = money(
+    firstMatch(text, [
+      /TOTAL\s*[:;.]?\s*\$?\s*([\d,]+\.\d{2})/i,
+      /SALDO\s+CORRIENTE[^\n]*?([\d,]+\.\d{2})/i,
+    ]),
+  );
+
+  // Leading zeros are print padding here too: the RECIBO prints `0457341` for
+  // what DATMEX.agua holds as `457341`.
+  const accountRef = account ? toDigits(account).replace(/^0+/, "") : null;
+  const cadastralKey = clave
+    ? clave.replace(/\s/g, "").slice(0, 2).toUpperCase() +
+      toDigits(clave.replace(/\s/g, "").slice(2))
+    : null;
+
+  return {
+    provider: "CESPT",
+    serviceKind: "WATER",
+    accountRef: accountRef || null,
+    cadastralKey: cadastralKey || null,
+    amount,
+    dueDate: parseDate(due),
+    period: null,
+    crossChecked: null,
+    notes,
+  };
+}
+
+// --- TELNOR (telephone) -----------------------------------------------------
+
+function parseTelnor(page: OcrPage): ParsedStatement {
+  const text = page.text;
+  const notes: string[] = [];
+
+  const label = firstMatch(text, [
+    /Tel[eé]fono\s*[:;.]?\s*([0-9OIlSBD]{3}\s?[0-9OIlSBD]{3}\s?[0-9OIlSBD]{4})/i,
+  ]);
+  // The payment stub prints phone (10 digits) + amount in cents (9) + a check
+  // digit: `6646093444 000099900 7` for a $999.00 bill. Reading the amount as
+  // 10 digits swallows the check digit and inflates the figure 100-fold.
+  const barcode = text.match(/\b(\d{10})(\d{9})\d\b/);
+
+  let accountRef: string | null = null;
+  let crossChecked: boolean | null = null;
+
+  // The bill prints the number with its 664 Tijuana LADA; DATMEX stores the
+  // bare local 7 digits, so the LADA is dropped rather than the stored value
+  // being padded — padding would guess at an area code for the 500+ existing
+  // rows that never recorded one.
+  if (label) accountRef = toDigits(label).slice(-7);
+  if (barcode) {
+    const fromBarcode = barcode[1].slice(-7);
+    if (accountRef) {
+      crossChecked = fromBarcode === accountRef;
+      if (!crossChecked) notes.push("el teléfono impreso no coincide con el código de barras");
+    } else {
+      accountRef = fromBarcode;
+      notes.push("teléfono leído del código de barras");
+    }
+  }
+
+  let amount = money(firstMatch(text, [/Total\s*a\s*Pagar\s*[:;.]?\s*\$?\s*([\d,]+\.?\d{0,2})/i]));
+  if (amount == null && barcode) {
+    amount = Number(barcode[2]) / 100;
+    notes.push("importe leído del código de barras");
+  }
+
+  return {
+    provider: "TELNOR",
+    serviceKind: "TELEPHONE",
+    accountRef: accountRef || null,
+    cadastralKey: null,
+    amount,
+    dueDate: parseDate(
+      firstMatch(text, [/Pagar\s*antes\s*de\s*[:;.]?\s*(\d{2}-\w{3}-\d{4})/i]),
+    ),
+    period: firstMatch(text, [/Mes\s*de\s*Facturaci[oó]n\s*[:;.]?\s*(\w+)/i]),
+    crossChecked,
+    notes,
+  };
+}
+
+const PARSERS: Record<string, (page: OcrPage) => ParsedStatement> = {
+  CFE: parseCfe,
+  CESPT: parseCespt,
+  TELNOR: parseTelnor,
+};
+
+const EMPTY: ParsedStatement = {
+  provider: null,
+  serviceKind: null,
+  accountRef: null,
+  cadastralKey: null,
+  amount: null,
+  dueDate: null,
+  period: null,
+  crossChecked: null,
+  notes: [],
+};
+
+/** Detect the provider and run its parser. */
+export function parseStatement(page: OcrPage): ParsedStatement {
+  const provider = detectProvider(page.text);
+  if (!provider) return { ...EMPTY, notes: ["no se reconoció el proveedor"] };
+  return PARSERS[provider](page);
+}