feat(statements): OCR intake for scanned utility bills
Staff key 300+ utility statements per company per month by hand. This adds the ingest -> split -> OCR -> match -> review pipeline that proposes customer and amount per page instead (RECEIPT_CAPTURE_SPEC §2), posting through the existing BillingService.createBatch seam with source=OCR and a per-document captureRef so machine and hand capture share one write path and audit trail. Everything was designed against 10 real scanned statements (46 pages of CFE, CESPT and Telnor bills) rather than from the sample-free spec. The scans have no text layer at all — they are camera images — so OCR is mandatory, and they arrive bundled one customer per page. Measured on those pages the parser identifies the provider 46/46 and reads an account reference 43/46; against the dev database that is 39/46 (85%) exact auto-match, 40/46 identified, with the rest genuine review cases. That closes the OCR-provider question in favour of self-hosted Tesseract: it clears the bar for a queue where a human confirms every row, and OcrProvider keeps a managed API a one-line swap. The samples corrected three things the spec had wrong or unknown: - Clave catastral is NOT predial. DATMEX.clave (934 rows) is what CESPT and predial bills print; DATMEX.predial, which PROPERTY_TAX.accountNumber holds, has 663 distinct values across 1135 rows and appears on no statement. The clave now lives on Property.cadastralKey as the matcher's secondary key; predial is left untouched. This had been blocking predial matching. - Gas was recoverable: 160 of 334 DATMEX.gas values are real account numbers (the rest are ESTACIONARIO/CILINDRO descriptors), now in GAS.meterNumber. - Phone is one billed line per property (534/18/1 across phone1/2/3), so the new TELEPHONE ServiceKind backfills from phone1 only, not three rows. Matching is scoped to one column per service kind and never reads the customer name — a CESPT receipt prints ARNAIZ ROSAS ELSA AURORA for an account this office holds under CATT, RANDY, because the printed name is the registrant, not the current owner. Where a provider prints a payment barcode it beats the printed label (one CFE label OCR'd a digit too many while its barcode was correct) and the two cross-check, with disagreement forcing review. Confirming a document whose service had no reference writes it back, so gas and any other cold start is a one-time cost rather than a permanent queue. Verified end to end against the live dev API and MinIO: real scans uploaded over HTTP, matched, confirmed against a check, and the resulting rows checked in MySQL (negative amounts, captureSource=OCR, concept derived from the batch kind, captureRef linking back to each page). Re-confirming a posted batch is refused. Test data was removed afterwards. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,199 @@
|
||||
import { Injectable } from "@nestjs/common";
|
||||
import type { ServiceKind } from "@jorgecuadros/database";
|
||||
import { PrismaService } from "../prisma/prisma.service";
|
||||
import type { ParsedStatement } from "./parsers/statement-parser";
|
||||
|
||||
export interface MatchResult {
|
||||
propertyServiceId: string | null;
|
||||
customerId: string | null;
|
||||
/** Why it landed here — shown in the review queue verbatim. */
|
||||
note: string;
|
||||
/** True only for an unambiguous hit on the scoped field. */
|
||||
confident: boolean;
|
||||
/** Populated when more than one service claims the same number. */
|
||||
candidates: { propertyServiceId: string; customerId: string; customerName: string }[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a parsed statement to the customer who should be billed for it.
|
||||
*
|
||||
* Two rules govern everything here.
|
||||
*
|
||||
* **Match on one scoped field, never fuzzily across all identifiers.** Each
|
||||
* service kind has exactly one column its statements print, and only that
|
||||
* column is consulted. A blanket search over accountNumber/meterNumber/route
|
||||
* would let a water account number collide with an unrelated phone number, and
|
||||
* the resulting mis-post would look perfectly ordinary in the ledger.
|
||||
*
|
||||
* **Never match on the customer name.** The name on a utility bill is the
|
||||
* account's registrant, which drifts from the current owner and is often years
|
||||
* stale — one sample CESPT receipt is printed to "ARNAIZ ROSAS ELSA AURORA"
|
||||
* for an account this office holds under "CATT, RANDY", who is not the same
|
||||
* person. Names are displayed for the reviewer to sanity-check, and are never
|
||||
* an input to matching.
|
||||
*/
|
||||
@Injectable()
|
||||
export class StatementMatcherService {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
/** Which PropertyService column a given kind's statements actually print. */
|
||||
private fieldFor(kind: ServiceKind): "accountNumber" | "meterNumber" | null {
|
||||
switch (kind) {
|
||||
case "ELECTRIC": // CFE "NO. DE SERVICIO" -> DATMEX.rpu
|
||||
case "WATER": // CESPT "Cuenta" / "No. DE CUENTA" -> DATMEX.agua
|
||||
case "TELEPHONE": // Telnor "Teléfono" (LADA stripped) -> DATMEX.telefono
|
||||
case "FEDERAL_ZONE":
|
||||
case "CABLE":
|
||||
return "accountNumber";
|
||||
case "GAS": // no account column in DATMEX; the number lived in notes
|
||||
return "meterNumber";
|
||||
// PROPERTY_TAX deliberately has no scoped column: what its
|
||||
// accountNumber holds is DATMEX.predial, which is neither unique nor
|
||||
// printed on any statement. Predial bills match on the clave catastral
|
||||
// alone — see matchByCadastralKey.
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async match(parsed: ParsedStatement, expectedKind: ServiceKind): Promise<MatchResult> {
|
||||
const kind = parsed.serviceKind ?? expectedKind;
|
||||
|
||||
// The uploader labels a batch with one service kind. If the parser reads a
|
||||
// page as a different provider, that is a mis-sorted page, not a match —
|
||||
// posting it would book a phone bill as a water charge.
|
||||
if (parsed.serviceKind && parsed.serviceKind !== expectedKind) {
|
||||
return this.unmatched(
|
||||
`la página parece de ${parsed.provider} (${parsed.serviceKind}) pero el lote es de ${expectedKind}`,
|
||||
);
|
||||
}
|
||||
|
||||
const field = this.fieldFor(kind);
|
||||
|
||||
if (field && parsed.accountRef) {
|
||||
const hit = await this.byServiceField(kind, field, parsed.accountRef);
|
||||
if (hit) return hit;
|
||||
}
|
||||
|
||||
// Secondary key. The clave catastral is printed on CESPT bills as well as
|
||||
// predial ones, so it rescues a page whose account number did not OCR —
|
||||
// which happened on real samples, where the clave read cleanly and the
|
||||
// account number did not.
|
||||
if (parsed.cadastralKey) {
|
||||
const hit = await this.byCadastralKey(kind, parsed.cadastralKey);
|
||||
if (hit) return hit;
|
||||
}
|
||||
|
||||
if (!field && !parsed.cadastralKey) {
|
||||
return this.unmatched(
|
||||
kind === "PROPERTY_TAX"
|
||||
? "el predial sólo se puede identificar por clave catastral y no se leyó ninguna"
|
||||
: `no hay campo de búsqueda definido para ${kind}`,
|
||||
);
|
||||
}
|
||||
return this.unmatched(
|
||||
parsed.accountRef
|
||||
? `no se encontró ningún servicio de ${kind} con la referencia ${parsed.accountRef}`
|
||||
: "no se pudo leer la referencia de la cuenta",
|
||||
);
|
||||
}
|
||||
|
||||
private async byServiceField(
|
||||
kind: ServiceKind,
|
||||
field: "accountNumber" | "meterNumber",
|
||||
ref: string,
|
||||
): Promise<MatchResult | null> {
|
||||
const rows = await this.prisma.propertyService.findMany({
|
||||
where: { kind, [field]: ref },
|
||||
select: {
|
||||
id: true,
|
||||
property: {
|
||||
select: { customerId: true, customer: { select: { name: true } } },
|
||||
},
|
||||
},
|
||||
});
|
||||
if (rows.length === 0) return null;
|
||||
|
||||
const candidates = rows.map((r) => ({
|
||||
propertyServiceId: r.id,
|
||||
customerId: r.property.customerId,
|
||||
customerName: r.property.customer.name,
|
||||
}));
|
||||
|
||||
// Duplicate account numbers do occur in the legacy data (the office's own
|
||||
// DUPLICADOS report existed for a reason), so every candidate is surfaced
|
||||
// for the reviewer to choose rather than one being picked arbitrarily.
|
||||
if (rows.length > 1) {
|
||||
return {
|
||||
propertyServiceId: null,
|
||||
customerId: null,
|
||||
note: `${rows.length} servicios comparten la referencia ${ref}`,
|
||||
confident: false,
|
||||
candidates,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
propertyServiceId: candidates[0].propertyServiceId,
|
||||
customerId: candidates[0].customerId,
|
||||
note: `coincidencia exacta por ${field === "accountNumber" ? "número de cuenta" : "medidor"} ${ref}`,
|
||||
confident: true,
|
||||
candidates,
|
||||
};
|
||||
}
|
||||
|
||||
private async byCadastralKey(
|
||||
kind: ServiceKind,
|
||||
key: string,
|
||||
): Promise<MatchResult | null> {
|
||||
const props = await this.prisma.property.findMany({
|
||||
where: { cadastralKey: key },
|
||||
select: {
|
||||
customerId: true,
|
||||
customer: { select: { name: true } },
|
||||
services: { where: { kind }, select: { id: true } },
|
||||
},
|
||||
});
|
||||
if (props.length === 0) return null;
|
||||
|
||||
const candidates = props.flatMap((p) =>
|
||||
(p.services.length ? p.services.map((s) => s.id) : [null]).map((sid) => ({
|
||||
propertyServiceId: sid as string,
|
||||
customerId: p.customerId,
|
||||
customerName: p.customer.name,
|
||||
})),
|
||||
);
|
||||
|
||||
if (candidates.length > 1) {
|
||||
return {
|
||||
propertyServiceId: null,
|
||||
customerId: null,
|
||||
note: `${candidates.length} propiedades comparten la clave catastral ${key}`,
|
||||
confident: false,
|
||||
candidates,
|
||||
};
|
||||
}
|
||||
|
||||
// The clave identifies the property with certainty, but it is a *secondary*
|
||||
// key: it was not the number the statement was issued against. Left for
|
||||
// review so the confirm also teaches the matcher the account number, rather
|
||||
// than the same page needing the fallback again next month.
|
||||
return {
|
||||
propertyServiceId: candidates[0].propertyServiceId ?? null,
|
||||
customerId: candidates[0].customerId,
|
||||
note: `identificado por clave catastral ${key}; confirme para registrar también el número de cuenta`,
|
||||
confident: false,
|
||||
candidates,
|
||||
};
|
||||
}
|
||||
|
||||
private unmatched(note: string): MatchResult {
|
||||
return {
|
||||
propertyServiceId: null,
|
||||
customerId: null,
|
||||
note,
|
||||
confident: false,
|
||||
candidates: [],
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user