Compare commits
12
Commits
v1.0.2
..
d6501f1d74
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d6501f1d74 | ||
|
|
216309190c | ||
|
|
e589bda28b | ||
|
|
98f7aa8a2d | ||
|
|
898cf48c80 | ||
|
|
70fe425043 | ||
|
|
567b033c46 | ||
|
|
1934470d53 | ||
|
|
fdbe9fdb88 | ||
|
|
e082113640 | ||
|
|
860d483bad | ||
|
|
783ec83464 |
@@ -37,6 +37,16 @@ env:
|
||||
jobs:
|
||||
build:
|
||||
name: Build ${{ matrix.image }}
|
||||
# release.yml pushes the release commit and its tag in a single `git push`,
|
||||
# so Gitea creates two runs for the same commit: one for master, one for the
|
||||
# tag. Only the tag run matters — it is the one that emits the X.Y.Z / X.Y
|
||||
# image tags, and it publishes `latest` and `sha-<short>` too, since it is
|
||||
# the same commit. Skip the branch run rather than racing or cancelling it.
|
||||
# Ordinary pushes to master (any message but `chore(release):`) still build.
|
||||
if: >-
|
||||
github.event_name != 'push' ||
|
||||
startsWith(github.ref, 'refs/tags/') ||
|
||||
!startsWith(github.event.head_commit.message, 'chore(release):')
|
||||
runs-on: docker
|
||||
container:
|
||||
image: docker:27-dind
|
||||
|
||||
@@ -152,7 +152,111 @@ jobs:
|
||||
|
||||
# One push for both refs: a commit that lands without its tag builds
|
||||
# nothing and looks like a successful release.
|
||||
git push origin "HEAD:master" "refs/tags/v${VERSION}"
|
||||
#
|
||||
# The output is captured because a failing *post-receive* hook does not
|
||||
# fail the push: git prints `remote: error: ...`, updates both refs and
|
||||
# exits 0. That is how v1.0.3 was cut — the hook 500'd, so Gitea never
|
||||
# created the build run, and this step went green anyway.
|
||||
if ! git push origin "HEAD:master" "refs/tags/v${VERSION}" 2>push.log; then
|
||||
cat push.log
|
||||
echo "::error::Push failed. Nothing was released."
|
||||
exit 1
|
||||
fi
|
||||
cat push.log
|
||||
|
||||
if grep -q '^remote: error' push.log; then
|
||||
echo "::warning::The remote's post-receive hook errored. Both refs landed,"
|
||||
echo "::warning::but Gitea most likely created no workflow run for them."
|
||||
echo "::warning::The next step checks and dispatches build.yml if needed."
|
||||
fi
|
||||
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
id: push
|
||||
|
||||
# Gitea creates workflow runs from the post-receive hook, so a hook error
|
||||
# silently costs you the build: the tag exists, no image is ever published,
|
||||
# and the failure only surfaces later as a 404 when deploy pulls the image.
|
||||
# Confirm the run exists; dispatch it if it does not; fail loudly if that
|
||||
# does not work either.
|
||||
- name: Verify build.yml started
|
||||
env:
|
||||
RELEASE_TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
VERSION: ${{ steps.ver.outputs.version }}
|
||||
SHA: ${{ steps.push.outputs.sha }}
|
||||
run: |
|
||||
node -e '
|
||||
const base = `${process.env.GITHUB_SERVER_URL}/api/v1/repos/${process.env.GITHUB_REPOSITORY}`;
|
||||
const headers = { Authorization: `token ${process.env.RELEASE_TOKEN}` };
|
||||
const sha = process.env.SHA;
|
||||
const tag = `v${process.env.VERSION}`;
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
// The master push and the tag push carry the SAME commit, so a sha
|
||||
// match alone is not enough: build.yml skips the master run by
|
||||
// design, and that skipped run would satisfy a sha-only check even
|
||||
// if the tag run were never created. When the API reports a ref for
|
||||
// the run, require it to be the tag; when it reports none, fall back
|
||||
// to the sha match rather than failing a release over a field name.
|
||||
const isTagRun = (r) => {
|
||||
const ref = r.head_branch || r.ref || "";
|
||||
return !ref || ref === tag || ref === `refs/tags/${tag}`;
|
||||
};
|
||||
|
||||
const started = async () => {
|
||||
const res = await fetch(`${base}/actions/runs?limit=30`, { headers });
|
||||
if (!res.ok) throw new Error(`runs query failed: HTTP ${res.status}`);
|
||||
const body = await res.json();
|
||||
return (body.workflow_runs || []).some(
|
||||
(r) =>
|
||||
r.head_sha === sha &&
|
||||
String(r.path || "").includes("build.yml") &&
|
||||
isTagRun(r),
|
||||
);
|
||||
};
|
||||
|
||||
// The hook fires synchronously with the push, so a run that is coming
|
||||
// is usually already there; the retries cover a busy instance.
|
||||
const poll = async (attempts) => {
|
||||
for (let i = 0; i < attempts; i++) {
|
||||
if (await started()) return true;
|
||||
await sleep(10_000);
|
||||
}
|
||||
return started();
|
||||
};
|
||||
|
||||
(async () => {
|
||||
if (await poll(3)) {
|
||||
console.log(`build.yml is running for ${sha}.`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`No build.yml run for ${sha}. Dispatching against ${tag}.`);
|
||||
const res = await fetch(
|
||||
`${base}/actions/workflows/build.yml/dispatches`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { ...headers, "Content-Type": "application/json" },
|
||||
// Must be the tag, not master: metadata-action only emits the
|
||||
// X.Y.Z and X.Y image tags when the ref is a semver tag. And
|
||||
// it must be the fully qualified ref — Gitea 404s on `v1.0.3`.
|
||||
body: JSON.stringify({ ref: `refs/tags/${tag}` }),
|
||||
},
|
||||
);
|
||||
if (!res.ok) console.log(`Dispatch returned HTTP ${res.status}.`);
|
||||
|
||||
if (await poll(3)) {
|
||||
console.log(`build.yml is running for ${sha}.`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`::error::${tag} is pushed but nothing is building it, and`);
|
||||
console.log(`::error::the dispatch did not take. Run "Build and Push Images"`);
|
||||
console.log(`::error::by hand with ref=${tag} (the tag, not master), then`);
|
||||
console.log(`::error::deploy. Check the Gitea server log for the`);
|
||||
console.log(`::error::post-receive error while you are at it.`);
|
||||
process.exit(1);
|
||||
})();
|
||||
'
|
||||
|
||||
- name: Summary
|
||||
env:
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
/** @type {import('jest').Config} */
|
||||
module.exports = {
|
||||
rootDir: "src",
|
||||
testEnvironment: "node",
|
||||
testRegex: ".*\\.spec\\.ts$",
|
||||
transform: { "^.+\\.ts$": "ts-jest" },
|
||||
};
|
||||
@@ -3,6 +3,7 @@
|
||||
"collection": "@nestjs/schematics",
|
||||
"sourceRoot": "src",
|
||||
"compilerOptions": {
|
||||
"deleteOutDir": true
|
||||
"deleteOutDir": true,
|
||||
"tsConfigPath": "tsconfig.build.json"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@jorgecuadros/api",
|
||||
"version": "1.0.2",
|
||||
"version": "1.0.5",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "nest build",
|
||||
|
||||
@@ -262,9 +262,26 @@ export class OpsService implements OnModuleInit {
|
||||
* deploy/scripts/pre-migrate-backup.mjs — the two write into the same volume
|
||||
* and both are listed as restore points by this same screen.
|
||||
*
|
||||
* --set-gtid-purged=OFF: the production server is the replication SOURCE with
|
||||
* GTID on, so without it every dump embeds SET @@GLOBAL.GTID_PURGED and is
|
||||
* unrestorable onto the very server it came from.
|
||||
* The dumper is probed at runtime rather than assumed. This command runs
|
||||
* inside the API image, whose `mysql-client` is Alpine's — i.e. MariaDB's —
|
||||
* where `mysqldump` is a deprecation-warning shim over `mariadb-dump` that
|
||||
* rejects --set-gtid-purged outright:
|
||||
* mysqldump: unknown variable 'set-gtid-purged=OFF'
|
||||
* which failed every backup, including the safety backups SYNC and REIMPORT
|
||||
* take first. MariaDB's dumper emits no GTID state unless asked (--gtid), so
|
||||
* there is nothing to suppress there; the flag is passed only when the dumper
|
||||
* on PATH advertises it, and the real binary is called directly only in the
|
||||
* MariaDB case (calling `mariadb-dump` whenever it merely exists would pick
|
||||
* it over a MySQL `mysqldump` earlier in PATH on a host carrying both).
|
||||
*
|
||||
* The probe is a command substitution, not `--help | grep -q`: PIPEFAIL is in
|
||||
* effect and grep closing the pipe early would make a supported flag look
|
||||
* unsupported.
|
||||
*
|
||||
* --set-gtid-purged=OFF (MySQL only): the production server is the
|
||||
* replication SOURCE with GTID on, so without it every dump embeds
|
||||
* SET @@GLOBAL.GTID_PURGED and is unrestorable onto the very server it came
|
||||
* from.
|
||||
*
|
||||
* The table-count assertion is not belt-and-braces: `gzip -t` passes on the
|
||||
* ~372-byte output of a mysqldump that died on its first statement, so a
|
||||
@@ -277,8 +294,12 @@ export class OpsService implements OnModuleInit {
|
||||
*/
|
||||
private dumpCommand(flags: string, db: string, out: string): string {
|
||||
return (
|
||||
`( mysqldump ${flags} --single-transaction --routines --triggers ` +
|
||||
`--no-tablespaces --set-gtid-purged=OFF ${db} | gzip -c > ${out} && ` +
|
||||
`DUMP=mysqldump; GTID=; ` +
|
||||
`case "$(mysqldump --help 2>/dev/null || true)" in ` +
|
||||
`*set-gtid-purged*) GTID=--set-gtid-purged=OFF;; ` +
|
||||
`*) command -v mariadb-dump >/dev/null 2>&1 && DUMP=mariadb-dump;; esac; ` +
|
||||
`( $DUMP ${flags} --single-transaction --routines --triggers ` +
|
||||
`--no-tablespaces $GTID ${db} | gzip -c > ${out} && ` +
|
||||
`gzip -t ${out} && ` +
|
||||
`TABLAS=$(gunzip -c ${out} | grep -c 'CREATE TABLE') && ` +
|
||||
`echo "tablas capturadas: $TABLAS" && ` +
|
||||
|
||||
@@ -6,8 +6,10 @@
|
||||
* The shipped implementation is self-hosted Tesseract (see tesseract.provider).
|
||||
* That choice is evidence-based rather than assumed: run against 46 pages of
|
||||
* real scanned CFE, CESPT and Telnor statements, it identified the provider on
|
||||
* 46/46 and extracted a usable account reference on 43/46, which is well past
|
||||
* the bar for a queue whose whole point is that a human confirms every row. A
|
||||
* 46/46 and extracted a usable account reference on 43/46, and on a later
|
||||
* corpus of 19 scanned municipal predial receipts it read the provider on
|
||||
* 19/19 and an identifier on 18/19 — well past the bar for a queue whose whole
|
||||
* point is that a human confirms every row. A
|
||||
* managed document-extraction API (Textract, Document Intelligence, Document
|
||||
* AI) fits behind this same interface if per-page accuracy ever proves
|
||||
* insufficient, with no schema change — but at 300+ pages/month/company it
|
||||
@@ -31,10 +33,10 @@ export interface OcrPage {
|
||||
/** Full page text, reading order, newline-separated. */
|
||||
text: string;
|
||||
/**
|
||||
* Word boxes. Needed because two of the three real layouts are *tables* —
|
||||
* the CESPT "RECIBO" prints `No. DE CUENTA` as a column header with the
|
||||
* value in the row beneath it, which line-oriented text cannot associate.
|
||||
* Parsers fall back to geometry for exactly those fields.
|
||||
* Word boxes. Needed because several of the real layouts are *tables* — the
|
||||
* CESPT "RECIBO" prints `No. DE CUENTA` as a column header with the value in
|
||||
* the row beneath it, which line-oriented text cannot associate. Parsers fall
|
||||
* back to geometry for exactly those fields.
|
||||
*/
|
||||
words: OcrWord[];
|
||||
/** Mean word confidence across the page, 0..1. */
|
||||
@@ -48,6 +50,22 @@ export interface OcrProvider {
|
||||
renderPages(pdf: Buffer): Promise<Buffer[]>;
|
||||
/** OCR a single rendered page image. */
|
||||
recognize(pageImage: Buffer): Promise<OcrPage>;
|
||||
/**
|
||||
* Read a PDF's own text layer, one entry per page, `null` where the page has
|
||||
* none worth using.
|
||||
*
|
||||
* Not every statement is a scan. The gas company e-mails born-digital CFDI
|
||||
* invoices whose text is already exact and already positioned — running those
|
||||
* through a rasteriser and a character recogniser can only lose information
|
||||
* (one sample turned `MEDIDOR: VM01014426` into `ar (LTR): 014420`) while
|
||||
* costing about a minute of CPU per page for the privilege. Where the layer
|
||||
* exists it is strictly better input for the same parsers, so it is tried
|
||||
* first and OCR remains the fallback for genuine scans.
|
||||
*
|
||||
* Positions are reported in the same pixel space `recognize` uses, so the
|
||||
* geometric helpers in the parsers work unchanged on either source.
|
||||
*/
|
||||
textPages(pdf: Buffer): Promise<(OcrPage | null)[]>;
|
||||
}
|
||||
|
||||
export const OCR_PROVIDER = Symbol("OCR_PROVIDER");
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import { parseBboxLayout } from "./tesseract.provider";
|
||||
|
||||
/**
|
||||
* Shaped like real `pdftotext -bbox-layout` output: the gas invoice lays its
|
||||
* header out as two columns of independent text flows, so poppler puts a label
|
||||
* and the value printed beside it in *different* `<line>` elements. Trusting
|
||||
* that grouping is what left `PERIODO FACTURADO` with no value next to it and
|
||||
* every period field empty on a batch whose text was perfectly readable.
|
||||
*/
|
||||
function word(x: number, y: number, text: string): string {
|
||||
return `<word xMin="${x}" yMin="${y}" xMax="${x + 20}" yMax="${y + 8}">${text}</word>`;
|
||||
}
|
||||
|
||||
function doc(...lines: string[]): string {
|
||||
return `<doc><page width="612" height="792">${lines
|
||||
.map((l) => `<flow><block><line>${l}</line></block></flow>`)
|
||||
.join("")}</page></doc>`;
|
||||
}
|
||||
|
||||
/** Enough words on the page to clear the "is this a real text layer" floor. */
|
||||
function padding(): string {
|
||||
return Array.from({ length: 50 }, (_, i) => word(10, 400 + i * 10, `w${i}`)).join("");
|
||||
}
|
||||
|
||||
describe("parseBboxLayout", () => {
|
||||
it("rejoins a label with the value printed beside it in another flow", () => {
|
||||
const [page] = parseBboxLayout(
|
||||
doc(
|
||||
word(20, 100, "PERIODO") + word(45, 100, "FACTURADO:"),
|
||||
word(300, 100.4, "20260630-20260630"),
|
||||
padding(),
|
||||
),
|
||||
1,
|
||||
);
|
||||
expect(page).not.toBeNull();
|
||||
expect(page!.text).toContain("PERIODO FACTURADO: 20260630-20260630");
|
||||
});
|
||||
|
||||
it("keeps genuinely separate lines apart", () => {
|
||||
const [page] = parseBboxLayout(
|
||||
doc(word(20, 100, "Cuenta:") + word(80, 100, "0900003463"), word(20, 130, "Nombre:"), padding()),
|
||||
1,
|
||||
);
|
||||
expect(page!.text.split("\n")).toContain("Cuenta: 0900003463");
|
||||
expect(page!.text.split("\n")).toContain("Nombre:");
|
||||
});
|
||||
|
||||
it("scales point coordinates into the render's pixel space", () => {
|
||||
// Word boxes have to land in the same coordinate space tesseract reports,
|
||||
// or the geometric helpers the parsers share silently stop finding values.
|
||||
const [page] = parseBboxLayout(doc(word(72, 144, "X") + padding()), 300 / 72);
|
||||
const x = page!.words.find((w) => w.text === "X")!;
|
||||
expect(x.left).toBeCloseTo(300);
|
||||
expect(x.top).toBeCloseTo(600);
|
||||
});
|
||||
|
||||
it("reports no text layer for a scan carrying a few stray glyphs", () => {
|
||||
expect(parseBboxLayout(doc(word(10, 10, "3") + word(40, 10, "of") + word(60, 10, "5")), 1)).toEqual([
|
||||
null,
|
||||
]);
|
||||
});
|
||||
|
||||
it("decodes the entities poppler escapes", () => {
|
||||
const [page] = parseBboxLayout(doc(word(10, 10, "A&B") + padding()), 1);
|
||||
expect(page!.text).toContain("A&B");
|
||||
});
|
||||
});
|
||||
@@ -105,6 +105,37 @@ export class TesseractOcrProvider implements OcrProvider {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* `pdftotext -bbox-layout` — the same poppler package `pdftoppm` comes from,
|
||||
* so this costs no extra dependency in the runtime image.
|
||||
*
|
||||
* A page is only accepted when it carries a real text layer. Scanned PDFs
|
||||
* frequently contain a handful of stray glyphs (a scanner watermark, a page
|
||||
* number stamped by the MFP), and treating those as the page's text would
|
||||
* hand every parser an almost-empty string and silently take OCR out of the
|
||||
* loop — so a floor of MIN_TEXT_WORDS words has to be present before the
|
||||
* layer is believed.
|
||||
*/
|
||||
async textPages(pdf: Buffer): Promise<(OcrPage | null)[]> {
|
||||
await this.require();
|
||||
return this.scratch(async (dir) => {
|
||||
const src = join(dir, "in.pdf");
|
||||
await writeFile(src, pdf);
|
||||
const out = join(dir, "out.html");
|
||||
try {
|
||||
await run("pdftotext", ["-bbox-layout", src, out]);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`pdftotext failed; falling back to OCR for this file: ${(err as Error).message}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
// Points to pixels at the render DPI, so word boxes from either source
|
||||
// land in one coordinate space and `valueUnder`'s thresholds hold.
|
||||
return parseBboxLayout(await readFile(out, "utf8"), this.dpi / 72);
|
||||
});
|
||||
}
|
||||
|
||||
async recognize(pageImage: Buffer): Promise<OcrPage> {
|
||||
await this.require();
|
||||
return this.scratch(async (dir) => {
|
||||
@@ -136,6 +167,130 @@ export class TesseractOcrProvider implements OcrProvider {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Below this many words a "text layer" is scanner debris, not a document.
|
||||
* The real born-digital samples carry 400+ words a page; the scanned ones
|
||||
* carry none at all, so the exact threshold is not delicate.
|
||||
*/
|
||||
const MIN_TEXT_WORDS = 40;
|
||||
|
||||
const ENTITIES: Record<string, string> = {
|
||||
amp: "&",
|
||||
lt: "<",
|
||||
gt: ">",
|
||||
quot: '"',
|
||||
apos: "'",
|
||||
};
|
||||
|
||||
function decodeEntities(s: string): string {
|
||||
return s.replace(/&(#x?[0-9a-fA-F]+|[a-z]+);/g, (whole, body: string) => {
|
||||
if (body[0] === "#") {
|
||||
const code =
|
||||
body[1] === "x" || body[1] === "X"
|
||||
? parseInt(body.slice(2), 16)
|
||||
: parseInt(body.slice(1), 10);
|
||||
return Number.isFinite(code) ? String.fromCodePoint(code) : whole;
|
||||
}
|
||||
return ENTITIES[body] ?? whole;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn `pdftotext -bbox-layout`'s XHTML into one OcrPage per PDF page.
|
||||
*
|
||||
* Parsed with regexes rather than an XML library on purpose: the output is
|
||||
* machine-generated by poppler with a fixed element shape (`page` > `flow` >
|
||||
* `block` > `line` > `word`), and the alternative is a parser dependency in
|
||||
* the API for one file format read in one place. Only `page` and `word` are
|
||||
* consulted — see below for why poppler's own `line` grouping is discarded.
|
||||
*
|
||||
* `confidence` is 1 for every word: these are the document's own characters,
|
||||
* not a recognition guess.
|
||||
*/
|
||||
export function parseBboxLayout(xhtml: string, scale: number): (OcrPage | null)[] {
|
||||
const pages: (OcrPage | null)[] = [];
|
||||
|
||||
for (const pageMatch of xhtml.matchAll(/<page\b[^>]*>([\s\S]*?)<\/page>/g)) {
|
||||
const words: OcrWord[] = [];
|
||||
|
||||
for (const w of pageMatch[1].matchAll(
|
||||
/<word\s+xMin="([\d.eE+-]+)"\s+yMin="([\d.eE+-]+)"\s+xMax="([\d.eE+-]+)"\s+yMax="([\d.eE+-]+)"\s*>([\s\S]*?)<\/word>/g,
|
||||
)) {
|
||||
const text = decodeEntities(w[5]).trim();
|
||||
if (!text) continue;
|
||||
const left = Number(w[1]) * scale;
|
||||
const top = Number(w[2]) * scale;
|
||||
words.push({
|
||||
text,
|
||||
left,
|
||||
top,
|
||||
width: Number(w[3]) * scale - left,
|
||||
height: Number(w[4]) * scale - top,
|
||||
confidence: 1,
|
||||
});
|
||||
}
|
||||
|
||||
pages.push(
|
||||
words.length >= MIN_TEXT_WORDS
|
||||
? { text: toVisualRows(words), words, confidence: 1 }
|
||||
: null,
|
||||
);
|
||||
}
|
||||
|
||||
return pages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reassemble words into the rows a reader sees, left to right.
|
||||
*
|
||||
* Poppler's own `<line>` grouping cannot be used for this. It groups by text
|
||||
* flow, and these invoices lay their fields out as two columns of independent
|
||||
* flows — so `PERIODO FACTURADO:` and the `20260630-20260630` printed beside
|
||||
* it end up in different `<line>` elements, and every label-then-value pattern
|
||||
* in the parsers misses a value that is plainly there on the page. Regrouping
|
||||
* by vertical position restores the adjacency, and matches what tesseract
|
||||
* hands back for the scanned version of the same layout.
|
||||
*
|
||||
* Rows are cut when a word's vertical centre leaves the band established by
|
||||
* the row's first word, which tolerates the sub-pixel baseline differences
|
||||
* between fonts on one line without merging two genuinely separate lines.
|
||||
*/
|
||||
function toVisualRows(words: OcrWord[]): string {
|
||||
const centre = (w: OcrWord) => w.top + w.height / 2;
|
||||
const sorted = [...words].sort((a, b) => centre(a) - centre(b) || a.left - b.left);
|
||||
|
||||
const rows: OcrWord[][] = [];
|
||||
let current: OcrWord[] = [];
|
||||
let band = 0;
|
||||
|
||||
for (const w of sorted) {
|
||||
if (!current.length) {
|
||||
current = [w];
|
||||
band = centre(w);
|
||||
continue;
|
||||
}
|
||||
// Half the word's own height: tall headings and body text both sit within
|
||||
// their own line's band, and neither reaches into the next one.
|
||||
if (Math.abs(centre(w) - band) <= Math.max(w.height, current[0].height) / 2) {
|
||||
current.push(w);
|
||||
} else {
|
||||
rows.push(current);
|
||||
current = [w];
|
||||
band = centre(w);
|
||||
}
|
||||
}
|
||||
if (current.length) rows.push(current);
|
||||
|
||||
return rows
|
||||
.map((r) =>
|
||||
[...r]
|
||||
.sort((a, b) => a.left - b.left)
|
||||
.map((w) => w.text)
|
||||
.join(" "),
|
||||
)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn tesseract's TSV into words plus reassembled text.
|
||||
*
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
import type { OcrPage } from "../ocr/ocr.provider";
|
||||
import {
|
||||
detectProvider,
|
||||
normalizeCadastralKey,
|
||||
parseStatement,
|
||||
} from "./statement-parser";
|
||||
|
||||
/**
|
||||
* Every string in this file is a verbatim excerpt of what the OCR engine
|
||||
* actually returned for a real receipt — misreads, dropped spaces, mangled
|
||||
* accents and all. That is the point: these are the specific ways these five
|
||||
* layouts have been observed to fail, and the assertions pin down what the
|
||||
* parser is supposed to do about each one. Inventing clean input here would
|
||||
* test nothing, because clean input was never the problem.
|
||||
*/
|
||||
function page(text: string): OcrPage {
|
||||
return { text, words: [], confidence: 0.9 };
|
||||
}
|
||||
|
||||
describe("detectProvider", () => {
|
||||
it("reads a Rosarito predial receipt as predial, not as a water bill", () => {
|
||||
// "Clave Catastral" is also a CESPT structural marker, so a predial page
|
||||
// whose header OCR'd badly must still not be claimed by the CESPT rule.
|
||||
expect(
|
||||
detectProvider(
|
||||
"e | Clave Catastral. KP-128-105 IMPUESTO PREDIAL ea rita\n" +
|
||||
"TASA | VALOR FISCAL | BIMESTRES | INCISO. | IMPUESTO",
|
||||
),
|
||||
).toBe("PREDIAL ROSARITO");
|
||||
});
|
||||
|
||||
it("keeps telling the three municipalities apart by their RFC", () => {
|
||||
expect(detectProvider("R.F.C. ATB-541201-KK2")).toBe("PREDIAL TIJUANA");
|
||||
expect(detectProvider("R.F.C. AMP-981201-HJ4")).toBe("PREDIAL ROSARITO");
|
||||
expect(detectProvider("MEN-540301-9J5")).toBe("PREDIAL ENSENADA");
|
||||
});
|
||||
|
||||
it("does not let the CFE rule claim a gas bill over 'PERIODO FACTURADO'", () => {
|
||||
expect(
|
||||
detectProvider("Orden de Facturación: 000009801640\nPERIODO FACTURADO: 20260630-20260630"),
|
||||
).toBe("GAS TIJUANA");
|
||||
});
|
||||
});
|
||||
|
||||
describe("normalizeCadastralKey", () => {
|
||||
it("keeps a letter in the third position instead of digitising it", () => {
|
||||
// `MMB01041` is a real key on file; mapping its B to 8 produced a key that
|
||||
// matches no property at all.
|
||||
expect(normalizeCadastralKey("MM-B01-041", [])).toBe("MMB01041");
|
||||
});
|
||||
|
||||
it("repairs the spurious I tesseract inserts into the prefix", () => {
|
||||
expect(normalizeCadastralKey("MIM-200-010", [])).toBe("MM200010");
|
||||
});
|
||||
|
||||
it("digitises confusable glyphs from position four onward", () => {
|
||||
expect(normalizeCadastralKey("KP-1O8-O45", [])).toBe("KP108045");
|
||||
});
|
||||
|
||||
it("flags a prefix it had to truncate", () => {
|
||||
const notes: string[] = [];
|
||||
expect(normalizeCadastralKey("KPX-128-106", notes)).toBe("KP128106");
|
||||
expect(notes).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parsePredialTijuana", () => {
|
||||
const TIJUANA = page(
|
||||
"Hats | AYUNTAMIENTO DE TIJUANA, BC $2,613.00 23/01/2026\n" +
|
||||
"y) TELEFONO: 973-7000 R.F.C. ATB-541201-KK2\n" +
|
||||
"ER AÑO VALOR FISCAL TASA IMPUESTO |CONCEPTO IMPORTE\n" +
|
||||
"ED ca 2026 1,207,15778 246 2,969.61 1102 - IMPUESTO PREDIAL 2,969.61\n" +
|
||||
"55164964310126000002613000054192\n" +
|
||||
"se 0 O (54427 [a] | TOTALAPAGAR: 2,613.00\n" +
|
||||
"Dc 1097 : FECHA VENCE : 31/ENE/2026",
|
||||
);
|
||||
|
||||
it("splits the payment barcode into account, deadline and amount", () => {
|
||||
const p = parseStatement(TIJUANA);
|
||||
expect(p.provider).toBe("PREDIAL TIJUANA");
|
||||
expect(p.serviceKind).toBe("PROPERTY_TAX");
|
||||
expect(p.accountRef).toBe("55164964");
|
||||
expect(p.amount).toBe(2613);
|
||||
expect(p.dueDate?.toISOString().slice(0, 10)).toBe("2026-01-31");
|
||||
expect(p.period).toBe("2026");
|
||||
});
|
||||
|
||||
it("reads the printed total even when the space in the label is lost", () => {
|
||||
// The real page OCR'd the label as "TOTALAPAGAR:", and it is that reading
|
||||
// that cross-checks the barcode's amount.
|
||||
expect(parseStatement(TIJUANA).crossChecked).toBe(true);
|
||||
});
|
||||
|
||||
it("refuses to trust a barcode the printed total contradicts", () => {
|
||||
const p = parseStatement(
|
||||
page(
|
||||
"R.F.C. ATB-541201-KK2\n" +
|
||||
"55164964310126000002613000054192\n" +
|
||||
"TOTAL A PAGAR: 9,613.00\nFECHA VENCE : 31/ENE/2026",
|
||||
),
|
||||
);
|
||||
expect(p.crossChecked).toBe(false);
|
||||
expect(p.notes.join(" ")).toContain("no coincide");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parsePredialRosarito", () => {
|
||||
it("takes the rounded Total, not the Sub Total printed above it", () => {
|
||||
const p = parseStatement(
|
||||
page(
|
||||
"AYUNTAMIENTO MUNICIPAL DE PLAYAS DE ROSARITO, B.C.\n" +
|
||||
"Ce Clave Catastral: + JR-400-008 7 | IMPUESTO PREDIAL\n" +
|
||||
"SUPERFICIE: 228.31 ZONA 30025 “Redondeo IT049 -$0.39 Sub Total $5,409.39\n" +
|
||||
"¿XTEMPORANEO DESPUES DE: 31/01/2026 Elaboro: MGLG\n" +
|
||||
"Total | $5,409.00\n" +
|
||||
"| Periodo por Pagar: 2026/1 2026/6",
|
||||
),
|
||||
);
|
||||
expect(p.cadastralKey).toBe("JR400008");
|
||||
expect(p.amount).toBe(5409);
|
||||
expect(p.dueDate?.toISOString().slice(0, 10)).toBe("2026-01-31");
|
||||
expect(p.period).toBe("2026");
|
||||
});
|
||||
|
||||
it("is not fooled by the unspaced 'SubTotal' spelling", () => {
|
||||
// This exact page read $9,624.85 off a receipt for $9,625.00 while the
|
||||
// lookbehind still assumed a space.
|
||||
const p = parseStatement(
|
||||
page(
|
||||
"AMP-981201-HJ4 IMPUESTO PREDIAL\n" +
|
||||
"SUPERFICIE. 367.62 ZONA:30151 | Redondco 17049 $0.15 SubTotal $9,624.85\n" +
|
||||
": Total | $9,625.00",
|
||||
),
|
||||
);
|
||||
expect(p.amount).toBe(9625);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parsePredialEnsenada", () => {
|
||||
const totals = (tail: string) =>
|
||||
page(
|
||||
"IMPRESION MAQUINA REGISTRADORA ez | MUNICIPIO DE ENSENADA\n" +
|
||||
"+7] DATOS. DEL.CAUSANTE alta A pe CLAVE MM-200-010 2 CUENTA\n" +
|
||||
`ES g € S| TOTALES 12,744.47 0.00 0.00 324.56 0.00 13,069.03 ${tail} |`,
|
||||
);
|
||||
|
||||
it("reads the paid total off the TOTALES row however the label OCR'd", () => {
|
||||
expect(parseStatement(totals("TOTA LA A $5,797.00")).amount).toBe(5797);
|
||||
expect(parseStatement(totals("orAL: M7 z] $14,414.00")).amount).toBe(14414);
|
||||
expect(parseStatement(totals("| TOTAL: = $6 246.00")).amount).toBe(6246);
|
||||
});
|
||||
|
||||
it("reports no amount rather than one whose $ was misread as an 8", () => {
|
||||
// `TOTAL: A 82,203.00` is a $2,203.00 receipt. Posting $82,203 would look
|
||||
// entirely ordinary in the ledger, so this page must go to review instead.
|
||||
const p = parseStatement(totals("TOTAL: A 82,203.00"));
|
||||
expect(p.amount).toBeNull();
|
||||
expect(p.notes.join(" ")).toContain("capturarlo a mano");
|
||||
});
|
||||
|
||||
it("never falls back to the assessed total on the same row", () => {
|
||||
expect(parseStatement(totals("yo: se TE= 58/4690]")).amount).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseGas", () => {
|
||||
const gas = (...cuentas: string[]) =>
|
||||
page(
|
||||
"GTI4608032K2 COMPAÑIA DE GAS DE TIJUANA\n" +
|
||||
"Fecha de Vencimiento: 2026/08/08\n" +
|
||||
cuentas.map((c) => `Cuenta: ${c}`).join("\n") +
|
||||
"\nPERIODO FACTURADO: 20260630-20260630\nTOTAL A PAGAR: $275.82",
|
||||
);
|
||||
|
||||
it("strips the printed leading zero to the stored account number", () => {
|
||||
const p = parseStatement(gas("0900003463", "0900003463", "0900003463"));
|
||||
expect(p.serviceKind).toBe("GAS");
|
||||
expect(p.accountRef).toBe("900003463");
|
||||
expect(p.amount).toBe(275.82);
|
||||
expect(p.dueDate?.toISOString().slice(0, 10)).toBe("2026-08-08");
|
||||
expect(p.period).toBe("2026-06");
|
||||
expect(p.crossChecked).toBe(true);
|
||||
});
|
||||
|
||||
it("takes the majority reading but still sends a disagreement to review", () => {
|
||||
const p = parseStatement(gas("0900003463", "0900003463", "0900003468"));
|
||||
expect(p.accountRef).toBe("900003463");
|
||||
expect(p.crossChecked).toBe(false);
|
||||
});
|
||||
|
||||
it("claims no cross-check from a single printing", () => {
|
||||
expect(parseStatement(gas("0900003463")).crossChecked).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -7,7 +7,11 @@ import type { OcrPage, OcrWord } from "../ocr/ocr.provider";
|
||||
* like with like and never has to know about provider-specific formatting.
|
||||
*/
|
||||
export interface ParsedStatement {
|
||||
/** "CFE" | "CESPT" | "TELNOR", or null when no parser claimed the page. */
|
||||
/**
|
||||
* "CFE" | "CESPT" | "TELNOR" | "GAS TIJUANA" | "PREDIAL TIJUANA" |
|
||||
* "PREDIAL ROSARITO" | "PREDIAL ENSENADA", or null when no parser claimed
|
||||
* the page.
|
||||
*/
|
||||
provider: string | null;
|
||||
serviceKind: ServiceKind | null;
|
||||
accountRef: string | null;
|
||||
@@ -90,6 +94,16 @@ function firstMatch(text: string, patterns: RegExp[]): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Every capture of `pattern` across the page, in order. */
|
||||
function allMatches(text: string, pattern: RegExp): string[] {
|
||||
const out: string[] = [];
|
||||
const re = new RegExp(pattern.source, pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`);
|
||||
for (const m of text.matchAll(re)) {
|
||||
if (m[1]) out.push(m[1].trim());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const MONTHS: Record<string, number> = {
|
||||
ENE: 0, FEB: 1, MAR: 2, ABR: 3, MAY: 4, JUN: 5,
|
||||
JUL: 6, AGO: 7, SEP: 8, OCT: 9, NOV: 10, DIC: 11,
|
||||
@@ -104,15 +118,16 @@ export function parseDate(raw: string | null | undefined): Date | null {
|
||||
let m = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/);
|
||||
if (m) return utc(+m[3], +m[2] - 1, +m[1]);
|
||||
|
||||
// 22-JUL-2026 / 22 JUN 26
|
||||
m = s.match(/^(\d{1,2})[-\s]([A-Z]{3})[A-Z]*[-\s](\d{2,4})$/);
|
||||
// 22-JUL-2026 / 22 JUN 26 / 31/ENE/2026 (Tijuana predial)
|
||||
m = s.match(/^(\d{1,2})[-\s/]([A-Z]{3})[A-Z]*[-\s/](\d{2,4})$/);
|
||||
if (m && MONTHS[m[2]] !== undefined) {
|
||||
const y = m[3].length === 2 ? 2000 + +m[3] : +m[3];
|
||||
return utc(y, MONTHS[m[2]], +m[1]);
|
||||
}
|
||||
|
||||
// 2026-07-22 (already normalised, e.g. decoded from a barcode)
|
||||
m = s.match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
||||
// 2026-07-22 (already normalised, e.g. decoded from a barcode) and the
|
||||
// 2026/08/08 the gas bill prints — same field order, different separator.
|
||||
m = s.match(/^(\d{4})[-/](\d{2})[-/](\d{2})$/);
|
||||
if (m) return utc(+m[1], +m[2] - 1, +m[3]);
|
||||
|
||||
return null;
|
||||
@@ -174,9 +189,30 @@ const BRAND: [string, RegExp][] = [
|
||||
["CFE", /comisi[oó]n federal de electricidad|CFE.?contigo|Suministrador de Servicios/i],
|
||||
["CESPT", /CESPT|COMISI[OÓ]N ESTATAL DE SERVICIOS/i],
|
||||
["TELNOR", /TELNOR|TELEFONOS DEL NOROESTE/i],
|
||||
["GAS TIJUANA", /COMPA[ÑN][IÍ]?A\s*DE\s*GAS\s*DE\s*TIJUANA|bajagas/i],
|
||||
// The municipal RFCs are the single most reliable discriminator on a predial
|
||||
// receipt: they are printed in a clean monospaced run on every layout, they
|
||||
// never change, and they say which of the three city treasuries issued the
|
||||
// page — which the wordmarks alone do not, since a Tijuana receipt also
|
||||
// carries "PLAYAS DE TIJUANA" and a Rosarito one "TIJUANA ENSENADA".
|
||||
["PREDIAL TIJUANA", /AYUNTAMIENTO\s*DE\s*TIJUANA|ATB.?541201/i],
|
||||
["PREDIAL ROSARITO", /AYUNTAMIENTO\s*MUNICIPAL\s*DE\s*PLAYAS\s*DE\s*ROSARITO|AMP.?981201|rosarito\.gob/i],
|
||||
["PREDIAL ENSENADA", /MUNICIPIO\s*DE\s*ENSENADA|MEN.?540301/i],
|
||||
];
|
||||
|
||||
/**
|
||||
* The predial rules come first because a Rosarito receipt prints "Clave
|
||||
* Catastral" as a boxed label — the very string the CESPT structural rule
|
||||
* looks for — so a page whose municipal header failed to OCR would otherwise
|
||||
* be claimed as a water bill and matched against the wrong column entirely.
|
||||
* "IMPUESTO PREDIAL" appears on all three municipal layouts and on none of the
|
||||
* utility ones, so it is the safe first question to ask.
|
||||
*/
|
||||
const LAYOUT: [string, RegExp][] = [
|
||||
["PREDIAL TIJUANA", /IMPUESTO\s*PREDIAL[\s\S]*?(?:CERTIFICACION\s*DE\s*CAJA|PASEO\s*DEL\s*CENTENARIO|PAGA\s*TU\s*PREDIAL)/i],
|
||||
["PREDIAL ENSENADA", /(?:IMPUESTO\s*PREDIAL[\s\S]*?TRANSPENINSULAR)|(?:IMPRESION\s*MAQUINA\s*REGISTRADORA)/i],
|
||||
["PREDIAL ROSARITO", /IMPUESTO\s*PREDIAL/i],
|
||||
["GAS TIJUANA", /Orden\s*de\s*Facturaci[oó]n|FACTOR\s*DE\s*PRESI[OÓ]N|GAS\s*LP/i],
|
||||
["CFE", /NO\.?\s*DE\s*SERVICIO|L[IÍ]MITE\s*DE\s*PAGO|PERIODO\s*FACTURADO/i],
|
||||
["CESPT", /SALDO\s+CORRIENTE|CLAVE\s*CATASTRAL|No\.?\s*DE\s*CUENTA/i],
|
||||
["TELNOR", /Mes\s*de\s*Facturaci[oó]n|Pagar\s*antes\s*de/i],
|
||||
@@ -364,10 +400,315 @@ function parseTelnor(page: OcrPage): ParsedStatement {
|
||||
};
|
||||
}
|
||||
|
||||
// --- GAS (Compañía de Gas de Tijuana / bajagas) ------------------------------
|
||||
|
||||
/**
|
||||
* These arrive as born-digital CFDI PDFs rather than scans, so the text layer
|
||||
* (see `TesseractOcrProvider.textPages`) usually reads them exactly and the
|
||||
* patterns below only have to be tolerant enough for the scanned case.
|
||||
*
|
||||
* The account number is printed three times — supply address, fiscal data, and
|
||||
* the payment stub at the foot — which is a free cross-check: three readings
|
||||
* that agree are near-certainly right, and any disagreement means one of them
|
||||
* was misread and the page deserves a human glance.
|
||||
*
|
||||
* `Cuenta` is what the matcher compares, not `Contrato`. The migration
|
||||
* recovered gas references out of `PropertyService.notes` into `meterNumber`
|
||||
* and what sat there is the 9-digit account (`900003463`), printed here with a
|
||||
* leading zero as `0900003463`.
|
||||
*/
|
||||
function parseGas(page: OcrPage): ParsedStatement {
|
||||
const text = page.text;
|
||||
const notes: string[] = [];
|
||||
|
||||
const seen = allMatches(text, /Cuenta\s*[:;.]?\s*([0-9OIlSBD]{6,12})/i).map((s) =>
|
||||
toDigits(s).replace(/^0+/, ""),
|
||||
);
|
||||
const distinct = [...new Set(seen.filter(Boolean))];
|
||||
|
||||
let accountRef: string | null = null;
|
||||
let crossChecked: boolean | null = null;
|
||||
if (distinct.length === 1) {
|
||||
accountRef = distinct[0];
|
||||
if (seen.length > 1) crossChecked = true;
|
||||
} else if (distinct.length > 1) {
|
||||
// Majority wins — the stub and the two address blocks print the same
|
||||
// number, so a single divergent reading is the misread one. It still goes
|
||||
// to review: `crossChecked: false` is what keeps the batch from
|
||||
// auto-matching a number one of three readings disagreed with.
|
||||
const tally = new Map<string, number>();
|
||||
for (const s of seen) tally.set(s, (tally.get(s) ?? 0) + 1);
|
||||
accountRef = [...tally.entries()].sort((a, b) => b[1] - a[1])[0][0];
|
||||
crossChecked = false;
|
||||
notes.push(`el número de cuenta se leyó de ${distinct.length} formas distintas (${distinct.join(", ")})`);
|
||||
}
|
||||
|
||||
const amount = money(
|
||||
firstMatch(text, [
|
||||
/TOTAL\s*A\s*PAGAR\s*[:;.]?\s*\$\s*([\d,]+\.\d{2})/i,
|
||||
/Total\s*a\s*pagar\s*[:;.]?\s*\$\s*([\d,]+\.\d{2})/i,
|
||||
]),
|
||||
);
|
||||
|
||||
// `20260630-20260630` — the range the bill was cut for. Both ends are the
|
||||
// same reading date on every sample, so the period is reported as the ISO
|
||||
// month rather than a range no ledger row would ever be searched by.
|
||||
const facturado = firstMatch(text, [/PERIODO\s*FACTURADO\s*[:;.]?\s*(\d{8})\s*-\s*\d{8}/i]);
|
||||
const period = facturado ? `${facturado.slice(0, 4)}-${facturado.slice(4, 6)}` : null;
|
||||
|
||||
return {
|
||||
provider: "GAS TIJUANA",
|
||||
serviceKind: "GAS",
|
||||
accountRef: accountRef || null,
|
||||
cadastralKey: null,
|
||||
amount,
|
||||
dueDate: parseDate(
|
||||
firstMatch(text, [/Fecha\s*de\s*Vencimiento\s*[:;.]?\s*(\d{4}\s*\/\s*\d{2}\s*\/\s*\d{2})/i])?.replace(
|
||||
/\s/g,
|
||||
"",
|
||||
),
|
||||
),
|
||||
period,
|
||||
crossChecked,
|
||||
notes,
|
||||
};
|
||||
}
|
||||
|
||||
// --- PREDIAL (municipal property tax) ---------------------------------------
|
||||
|
||||
/**
|
||||
* Normalise a printed clave catastral to the eight-character form
|
||||
* `Property.cadastralKey` holds. The municipalities print it grouped
|
||||
* (`KP-128-106`, `MM-B01-041`); the stored value drops the separators
|
||||
* (`KP128106`, `MMB01041`).
|
||||
*
|
||||
* The shape is *not* two letters and six digits, which is the assumption that
|
||||
* has to be resisted here. Across the 932 distinct claves on file, characters
|
||||
* four through eight are digits without exception, but the third is a digit in
|
||||
* 917 of them and one of `A`, `B`, `H`, `T` in the other fifteen. Running the
|
||||
* whole tail through `toDigits` — which maps `B` to `8` — is what turned a real
|
||||
* `MMB01041` into a nonexistent `MM801041`, so only positions four onward get
|
||||
* that treatment and a letter in the third position is kept as printed.
|
||||
*
|
||||
* That leaves a genuine ambiguity at that one position: a `B` there might be a
|
||||
* misread `8`, and 34 stored claves do carry an `8` there against six with a
|
||||
* `B`. It is left as read rather than guessed, because a page that fails to
|
||||
* match lands in the review queue where a human fixes it in seconds, while a
|
||||
* page that matches the wrong property posts a charge to the wrong customer.
|
||||
*
|
||||
* The two-letter prefix is the other fragile part. Tesseract inserts a spurious
|
||||
* `I` into letter pairs with some regularity — a real `MM-200-010` came back as
|
||||
* `MIM-200-010` — so a run longer than two letters has its `I`/`L` dropped
|
||||
* first, which recovers exactly that case. Anything still not two letters is
|
||||
* truncated and flagged, because a wrong prefix silently matches the wrong
|
||||
* property or, more often, nothing at all.
|
||||
*/
|
||||
export function normalizeCadastralKey(
|
||||
raw: string,
|
||||
notes: string[],
|
||||
): string | null {
|
||||
const m = raw.match(/^([A-Za-z|]{2,5})[-\s]?([A-Za-z0-9|]{3})[-\s]?([0-9OIlSBD]{3})$/);
|
||||
if (!m) return null;
|
||||
|
||||
let letters = m[1].toUpperCase().replace(/[^A-Z]/g, "");
|
||||
if (letters.length > 2) {
|
||||
const stripped = letters.replace(/[IL]/g, "");
|
||||
if (stripped.length === 2) {
|
||||
letters = stripped;
|
||||
} else {
|
||||
letters = letters.slice(0, 2);
|
||||
notes.push(`la clave catastral se leyó como "${m[1]}"; se tomó "${letters}"`);
|
||||
}
|
||||
}
|
||||
if (letters.length !== 2) return null;
|
||||
|
||||
const third = m[2][0].toUpperCase();
|
||||
const tail =
|
||||
(/[A-Z]/.test(third) ? third : toDigits(third)) +
|
||||
toDigits(m[2].slice(1)) +
|
||||
toDigits(m[3]);
|
||||
|
||||
return tail.length === 6 ? letters + tail : null;
|
||||
}
|
||||
|
||||
/** The grouped clave as printed, anchored to its label when one survived OCR. */
|
||||
const GROUPED_CLAVE = "[A-Z|]{2,5}-[A-Z0-9OIlSBD]{3}-[0-9OIlSBD]{3}";
|
||||
|
||||
function findCadastralKey(text: string, notes: string[]): string | null {
|
||||
const labelled = firstMatch(text, [
|
||||
new RegExp(`Clave\\s*Catastral\\s*[^A-Z0-9]{0,8}(${GROUPED_CLAVE})`, "i"),
|
||||
new RegExp(`CLAVE\\s*[^A-Z0-9]{0,8}(${GROUPED_CLAVE})`, "i"),
|
||||
]);
|
||||
if (labelled) return normalizeCadastralKey(labelled, notes);
|
||||
|
||||
// Ensenada's label ("CLAVE") lands inside a table header that OCRs into
|
||||
// noise more often than not, so the bare grouped shape is accepted as a
|
||||
// fallback. It is distinctive enough — two letters and two three-character
|
||||
// groups joined by hyphens appears nowhere else on these pages.
|
||||
const bare = firstMatch(text, [new RegExp(`\\b(${GROUPED_CLAVE})\\b`)]);
|
||||
return bare ? normalizeCadastralKey(bare, notes) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tijuana: a "CERTIFICACIÓN DE CAJA" whose payment barcode is one 32-digit run
|
||||
* of `account(8) + due date(DDMMYY) + amount(9) + folio(9)`, verified against
|
||||
* all five sample pages. Municipal totals are whole pesos (the receipt itself
|
||||
* carries a "Redondeo" line), so the barcode amount needs no decimal point.
|
||||
*
|
||||
* No clave catastral is printed anywhere on this layout — the 8-digit
|
||||
* municipal account is the only identifier, and it is not a number the legacy
|
||||
* database ever held. Until a reviewer confirms one, every Tijuana page lands
|
||||
* in review; confirming teaches the matcher (see `learnAccountRefs`) so the
|
||||
* same property matches itself next year.
|
||||
*/
|
||||
function parsePredialTijuana(page: OcrPage): ParsedStatement {
|
||||
const text = page.text;
|
||||
const notes: string[] = [];
|
||||
|
||||
const barcode = text.match(/(?<![0-9OIlSBD])([0-9OIlSBD]{32})(?![0-9OIlSBD])/);
|
||||
const printedTotal = money(
|
||||
firstMatch(text, [/TOTAL\s*A?\s*PAGAR\s*[:;.]?\s*\$?\s*([\d,]+\.?\d{0,2})/i]),
|
||||
);
|
||||
|
||||
let accountRef: string | null = null;
|
||||
let amount: number | null = printedTotal;
|
||||
let dueDate: Date | null = null;
|
||||
let crossChecked: boolean | null = null;
|
||||
|
||||
if (barcode) {
|
||||
const run = toDigits(barcode[1]);
|
||||
const d = run.slice(8, 14);
|
||||
const fromBarcode = Number(run.slice(14, 23));
|
||||
accountRef = run.slice(0, 8);
|
||||
dueDate = parseDate(`20${d.slice(4, 6)}-${d.slice(2, 4)}-${d.slice(0, 2)}`);
|
||||
notes.push("cuenta, importe y vencimiento leídos del código de barras");
|
||||
|
||||
if (printedTotal != null) {
|
||||
// Guarding the money, not the account number: the printed total is the
|
||||
// figure a human would key, so when the two disagree one of them is a
|
||||
// misread peso amount and nothing should post unreviewed.
|
||||
crossChecked = Math.abs(printedTotal - fromBarcode) < 0.5;
|
||||
if (!crossChecked) {
|
||||
notes.push(
|
||||
`el total impreso (${printedTotal}) no coincide con el código de barras (${fromBarcode})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (amount == null) amount = fromBarcode;
|
||||
}
|
||||
|
||||
if (!dueDate) {
|
||||
dueDate = parseDate(
|
||||
firstMatch(text, [/FECHA\s*VENCE\s*[:;.]?\s*(\d{1,2}\/\w{3}\/\d{4})/i]),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
provider: "PREDIAL TIJUANA",
|
||||
serviceKind: "PROPERTY_TAX",
|
||||
accountRef: accountRef || null,
|
||||
cadastralKey: null,
|
||||
amount,
|
||||
dueDate,
|
||||
// The fiscal year, which is what the legacy ledger's `period` holds for
|
||||
// predial ("2026" is its single most common value). It is read from the
|
||||
// assessment table's year column, and failing that from the deadline: a
|
||||
// predial bill for year N falls due on 31 January of year N.
|
||||
period:
|
||||
firstMatch(text, [/VALOR\s*FISCAL[\s\S]{0,160}?\b(20\d{2})\b/i]) ??
|
||||
(dueDate ? String(dueDate.getUTCFullYear()) : null),
|
||||
crossChecked,
|
||||
notes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Rosarito: a wide "CERTIFICACIÓN DE CAJA" keyed by clave catastral, with no
|
||||
* account number of its own — the clave is the identifier, which is exactly
|
||||
* what `Property.cadastralKey` holds, so these match on the first pass.
|
||||
*
|
||||
* The total is read with a negative lookbehind on "Sub": the receipt prints
|
||||
* `Sub Total $5,409.39` (before the peso rounding) directly above
|
||||
* `Total $5,409.00`, and taking the first "Total" on the page books 39 cents
|
||||
* that the municipality did not charge. The lookbehind allows zero spaces
|
||||
* because the label prints both ways — `Sub Total` on one sample and
|
||||
* `SubTotal` on the next, and the tight one is what slipped past a fixed
|
||||
* `Sub\s` and read $9,624.85 off a receipt for $9,625.00.
|
||||
*/
|
||||
function parsePredialRosarito(page: OcrPage): ParsedStatement {
|
||||
const notes: string[] = [];
|
||||
const text = page.text;
|
||||
|
||||
return {
|
||||
provider: "PREDIAL ROSARITO",
|
||||
serviceKind: "PROPERTY_TAX",
|
||||
accountRef: null,
|
||||
cadastralKey: findCadastralKey(text, notes),
|
||||
amount: money(firstMatch(text, [/(?<!Sub\s{0,3})Total\s*[|:;.]?\s*\$\s*([\d,]+\.\d{2})/i])),
|
||||
// "EXTEMPORANEO DESPUES DE: 31/01/2026" — the leading E is regularly eaten
|
||||
// by the box rule printed over it, so the anchor starts at "XTEMPORANEO".
|
||||
dueDate: parseDate(
|
||||
firstMatch(text, [/XTEMPOR[AÁ]NEO\s*DESPU[EÉ]S\s*DE\s*[:;.]?\s*(\d{2}\/\d{2}\/\d{4})/i]),
|
||||
),
|
||||
period: firstMatch(text, [/Periodo\s*por\s*Pagar\s*[:;.]?\s*(20\d{2})/i]),
|
||||
crossChecked: null,
|
||||
notes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensenada: a dot-matrix "IMPRESION MAQUINA REGISTRADORA" statement, by some
|
||||
* distance the worst-scanning of the three. Matching is by clave catastral.
|
||||
*
|
||||
* The amount is read positionally rather than by label, because the label does
|
||||
* not survive: across five real pages the same word came back as `TOTAL:`,
|
||||
* `TOTA LA A` and `orAL:`. What is stable is the row — the summary line that
|
||||
* starts `TOTALES` carries the assessed figures across it and the amount
|
||||
* actually paid last, at the right margin.
|
||||
*
|
||||
* That last figure must carry a literal `$`. On a real sample the paid total
|
||||
* printed as `TOTAL: A $2,203.00` and OCR'd as `TOTAL: A 82,203.00` — the
|
||||
* dollar sign read as an 8, a mistake that would post a $2,203 charge as
|
||||
* $82,203 and look entirely ordinary in the ledger. Requiring the `$` costs
|
||||
* that page its amount and sends it to review, which is the only acceptable
|
||||
* failure here. The unprefixed figures earlier on the row are deliberately not
|
||||
* a fallback: they are the tax assessed before the early-payment discount, not
|
||||
* what was paid.
|
||||
*/
|
||||
function parsePredialEnsenada(page: OcrPage): ParsedStatement {
|
||||
const notes: string[] = [];
|
||||
const text = page.text;
|
||||
|
||||
const totalsRow = text.split("\n").find((l) => /TOTALES/i.test(l)) ?? "";
|
||||
const figures = allMatches(totalsRow, /\$\s*(\d[\d,.\s]*\.\d{2})/);
|
||||
const amount = figures.length ? money(figures[figures.length - 1]) : null;
|
||||
if (amount == null) {
|
||||
notes.push("no se pudo leer el importe con certeza; capturarlo a mano");
|
||||
}
|
||||
|
||||
return {
|
||||
provider: "PREDIAL ENSENADA",
|
||||
serviceKind: "PROPERTY_TAX",
|
||||
accountRef: null,
|
||||
cadastralKey: findCadastralKey(text, notes),
|
||||
amount,
|
||||
// This layout prints no payment deadline at all — it is a receipt for a
|
||||
// payment already made at the municipal window.
|
||||
dueDate: null,
|
||||
period: firstMatch(text, [/A[ÑN]O\s*[\s\S]{0,60}?\b(20\d{2})\b/i]),
|
||||
crossChecked: null,
|
||||
notes,
|
||||
};
|
||||
}
|
||||
|
||||
const PARSERS: Record<string, (page: OcrPage) => ParsedStatement> = {
|
||||
CFE: parseCfe,
|
||||
CESPT: parseCespt,
|
||||
TELNOR: parseTelnor,
|
||||
"GAS TIJUANA": parseGas,
|
||||
"PREDIAL TIJUANA": parsePredialTijuana,
|
||||
"PREDIAL ROSARITO": parsePredialRosarito,
|
||||
"PREDIAL ENSENADA": parsePredialEnsenada,
|
||||
};
|
||||
|
||||
const EMPTY: ParsedStatement = {
|
||||
|
||||
@@ -32,30 +32,46 @@ export interface MatchResult {
|
||||
* person. Names are displayed for the reviewer to sanity-check, and are never
|
||||
* an input to matching.
|
||||
*/
|
||||
/**
|
||||
* Which `PropertyService` column a given kind's statements actually print.
|
||||
*
|
||||
* Exported because the same answer governs three places that must agree: the
|
||||
* lookup here, the blank-service fill on review, and the write-back on confirm.
|
||||
* When they disagree, a reference gets learned into a column nothing searches,
|
||||
* and the same page returns to the review queue every month forever.
|
||||
*
|
||||
* `meterNumber` is doing double duty for the two kinds whose printed reference
|
||||
* DATMEX never held in `accountNumber`:
|
||||
* - GAS, where the number lived in free-text notes, and
|
||||
* - PROPERTY_TAX, where `accountNumber` holds DATMEX.predial — a 3-4 digit
|
||||
* office file number that is neither unique nor printed on any statement.
|
||||
* The Tijuana municipal receipt prints an 8-digit account and no clave
|
||||
* catastral at all, so it needs a column of its own; overwriting the legacy
|
||||
* predial numbers to make room would destroy the only link back to the
|
||||
* original records.
|
||||
*/
|
||||
export function scopedRefField(
|
||||
kind: ServiceKind,
|
||||
): "accountNumber" | "meterNumber" | null {
|
||||
switch (kind) {
|
||||
case "ELECTRIC": // CFE "NO. DE SERVICIO" -> DATMEX.rpu
|
||||
case "WATER": // CESPT "Cuenta" / "No. DE CUENTA" -> DATMEX.agua
|
||||
case "TELEPHONE": // Telnor "Teléfono" (LADA stripped) -> DATMEX.telefono
|
||||
case "FEDERAL_ZONE":
|
||||
case "CABLE":
|
||||
return "accountNumber";
|
||||
case "GAS": // bajagas "Cuenta" -> recovered from notes into meterNumber
|
||||
case "PROPERTY_TAX": // Tijuana's 8-digit municipal account
|
||||
return "meterNumber";
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class StatementMatcherService {
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
/** Which PropertyService column a given kind's statements actually print. */
|
||||
private fieldFor(kind: ServiceKind): "accountNumber" | "meterNumber" | null {
|
||||
switch (kind) {
|
||||
case "ELECTRIC": // CFE "NO. DE SERVICIO" -> DATMEX.rpu
|
||||
case "WATER": // CESPT "Cuenta" / "No. DE CUENTA" -> DATMEX.agua
|
||||
case "TELEPHONE": // Telnor "Teléfono" (LADA stripped) -> DATMEX.telefono
|
||||
case "FEDERAL_ZONE":
|
||||
case "CABLE":
|
||||
return "accountNumber";
|
||||
case "GAS": // no account column in DATMEX; the number lived in notes
|
||||
return "meterNumber";
|
||||
// PROPERTY_TAX deliberately has no scoped column: what its
|
||||
// accountNumber holds is DATMEX.predial, which is neither unique nor
|
||||
// printed on any statement. Predial bills match on the clave catastral
|
||||
// alone — see matchByCadastralKey.
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async match(parsed: ParsedStatement, expectedKind: ServiceKind): Promise<MatchResult> {
|
||||
const kind = parsed.serviceKind ?? expectedKind;
|
||||
|
||||
@@ -68,33 +84,39 @@ export class StatementMatcherService {
|
||||
);
|
||||
}
|
||||
|
||||
const field = this.fieldFor(kind);
|
||||
const field = scopedRefField(kind);
|
||||
|
||||
if (field && parsed.accountRef) {
|
||||
const hit = await this.byServiceField(kind, field, parsed.accountRef);
|
||||
if (hit) return hit;
|
||||
}
|
||||
|
||||
// Secondary key. The clave catastral is printed on CESPT bills as well as
|
||||
// predial ones, so it rescues a page whose account number did not OCR —
|
||||
// which happened on real samples, where the clave read cleanly and the
|
||||
// account number did not.
|
||||
// The clave catastral is printed on CESPT bills as well as predial ones, so
|
||||
// it rescues a page whose account number did not OCR — which happened on
|
||||
// real samples, where the clave read cleanly and the account number did
|
||||
// not. On the Rosarito and Ensenada predial layouts it is not a rescue at
|
||||
// all but the only identifier the receipt carries, so a unique hit there is
|
||||
// as good as any account-number match and is treated as one.
|
||||
if (parsed.cadastralKey) {
|
||||
const hit = await this.byCadastralKey(kind, parsed.cadastralKey);
|
||||
const primary = kind === "PROPERTY_TAX" && !parsed.accountRef;
|
||||
const hit = await this.byCadastralKey(kind, parsed.cadastralKey, primary);
|
||||
if (hit) return hit;
|
||||
}
|
||||
|
||||
if (!field && !parsed.cadastralKey) {
|
||||
return this.unmatched(`no hay campo de búsqueda definido para ${kind}`);
|
||||
}
|
||||
if (!parsed.accountRef && !parsed.cadastralKey) {
|
||||
return this.unmatched(
|
||||
kind === "PROPERTY_TAX"
|
||||
? "el predial sólo se puede identificar por clave catastral y no se leyó ninguna"
|
||||
: `no hay campo de búsqueda definido para ${kind}`,
|
||||
? "no se leyó ni la clave catastral ni la cuenta municipal"
|
||||
: "no se pudo leer la referencia de la cuenta",
|
||||
);
|
||||
}
|
||||
return this.unmatched(
|
||||
parsed.accountRef
|
||||
? `no se encontró ningún servicio de ${kind} con la referencia ${parsed.accountRef}`
|
||||
: "no se pudo leer la referencia de la cuenta",
|
||||
: `no se encontró ninguna propiedad con la clave catastral ${parsed.cadastralKey}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -145,6 +167,8 @@ export class StatementMatcherService {
|
||||
private async byCadastralKey(
|
||||
kind: ServiceKind,
|
||||
key: string,
|
||||
/** True when the clave is the identifier the statement was issued against. */
|
||||
primary: boolean,
|
||||
): Promise<MatchResult | null> {
|
||||
const props = await this.prisma.property.findMany({
|
||||
where: { cadastralKey: key },
|
||||
@@ -174,15 +198,21 @@ export class StatementMatcherService {
|
||||
};
|
||||
}
|
||||
|
||||
// The clave identifies the property with certainty, but it is a *secondary*
|
||||
// key: it was not the number the statement was issued against. Left for
|
||||
// review so the confirm also teaches the matcher the account number, rather
|
||||
// than the same page needing the fallback again next month.
|
||||
// When the clave is the *secondary* key — a utility bill that also happens
|
||||
// to print it — the page is left for review, because the clave was not the
|
||||
// number the statement was issued against and confirming is what teaches
|
||||
// the matcher the account number for next month. When it is the primary key
|
||||
// (Rosarito and Ensenada predial, which print nothing else), a unique hit
|
||||
// is a real match and there is no second number to learn.
|
||||
return {
|
||||
propertyServiceId: candidates[0].propertyServiceId ?? null,
|
||||
customerId: candidates[0].customerId,
|
||||
note: `identificado por clave catastral ${key}; confirme para registrar también el número de cuenta`,
|
||||
confident: false,
|
||||
note: primary
|
||||
? `coincidencia exacta por clave catastral ${key}`
|
||||
: `identificado por clave catastral ${key}; confirme para registrar también el número de cuenta`,
|
||||
// A clave with no service row of the right kind behind it still needs a
|
||||
// human: there is nothing to attach the posting to.
|
||||
confident: primary && candidates[0].propertyServiceId != null,
|
||||
candidates,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import { BillingService } from "../billing/billing.service";
|
||||
import type { UploadedFileLike } from "../storage/upload-file";
|
||||
import { OCR_PROVIDER, type OcrProvider } from "./ocr/ocr.provider";
|
||||
import { parseStatement } from "./parsers/statement-parser";
|
||||
import { StatementMatcherService } from "./statement-matcher.service";
|
||||
import { StatementMatcherService, scopedRefField } from "./statement-matcher.service";
|
||||
import type { ConfirmBatchDto, ReviewDocumentDto } from "./statement.dto";
|
||||
|
||||
/**
|
||||
@@ -127,14 +127,23 @@ export class StatementsService {
|
||||
await this.storage.put(sourceKey, file.buffer, "application/pdf");
|
||||
|
||||
const pages = await this.ocr.renderPages(file.buffer);
|
||||
for (const image of pages) {
|
||||
// Page images are still rendered and stored for every file, text layer or
|
||||
// not: the review screen shows the reviewer the page, and "what the
|
||||
// parser read" is only checkable against a picture of the paper.
|
||||
const textLayer = await this.ocr.textPages(file.buffer).catch(() => []);
|
||||
|
||||
for (const [index, image] of pages.entries()) {
|
||||
pageNumber += 1;
|
||||
const storageKey = `statement/${batchId}/page-${pageNumber}.png`;
|
||||
await this.storage.put(storageKey, image, "image/png");
|
||||
|
||||
try {
|
||||
const ocr = await this.ocr.recognize(image);
|
||||
const embedded = textLayer[index] ?? null;
|
||||
const ocr = embedded ?? (await this.ocr.recognize(image));
|
||||
const parsed = parseStatement(ocr);
|
||||
if (embedded) {
|
||||
parsed.notes.unshift("texto leído del PDF original, sin OCR");
|
||||
}
|
||||
const match = await this.matcher.match(parsed, serviceKind);
|
||||
|
||||
const notes = [...parsed.notes, match.note].filter(Boolean);
|
||||
@@ -295,8 +304,8 @@ export class StatementsService {
|
||||
where: { id: doc.batchId },
|
||||
select: { serviceKind: true },
|
||||
});
|
||||
if (batch) {
|
||||
const field = batch.serviceKind === "GAS" ? "meterNumber" : "accountNumber";
|
||||
const field = batch && scopedRefField(batch.serviceKind);
|
||||
if (batch && field) {
|
||||
const blank = await this.prisma.propertyService.findMany({
|
||||
where: {
|
||||
kind: batch.serviceKind,
|
||||
@@ -434,7 +443,8 @@ export class StatementsService {
|
||||
docs: { matchedPropertyServiceId: string | null; extractedAccountRef: string | null }[],
|
||||
kind: ServiceKind,
|
||||
) {
|
||||
const field = kind === "GAS" ? "meterNumber" : "accountNumber";
|
||||
const field = scopedRefField(kind);
|
||||
if (!field) return;
|
||||
for (const d of docs) {
|
||||
if (!d.matchedPropertyServiceId || !d.extractedAccountRef) continue;
|
||||
await this.prisma.propertyService.updateMany({
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"exclude": ["node_modules", "dist", "**/*.spec.ts"]
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@jorgecuadros/web",
|
||||
"version": "1.0.2",
|
||||
"version": "1.0.5",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev -p 4500",
|
||||
|
||||
@@ -837,6 +837,46 @@ button {
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
/* Upload progress (Operaciones ingest) */
|
||||
.upload-progress {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 6px;
|
||||
padding: 4px 0 8px;
|
||||
}
|
||||
.progress-track {
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
height: 8px;
|
||||
border-radius: 999px;
|
||||
background: var(--paper-2);
|
||||
}
|
||||
.progress-fill {
|
||||
height: 100%;
|
||||
border-radius: 999px;
|
||||
background: var(--brand-500);
|
||||
transition: width 0.2s linear;
|
||||
}
|
||||
.progress-indeterminate .progress-fill {
|
||||
width: 40% !important;
|
||||
animation: progress-slide 1.2s var(--ease-out-quart) infinite;
|
||||
}
|
||||
@keyframes progress-slide {
|
||||
0% {
|
||||
transform: translateX(-100%);
|
||||
}
|
||||
100% {
|
||||
transform: translateX(250%);
|
||||
}
|
||||
}
|
||||
.upload-progress-stats {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
@keyframes shimmer {
|
||||
0% {
|
||||
background-position: -420px 0;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { Fragment, useCallback, useEffect, useRef, useState } from "react";
|
||||
import { AppShell } from "@/components/AppShell";
|
||||
import { useCan } from "@/lib/abilities";
|
||||
import {
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
startOpsJob,
|
||||
uploadIngest,
|
||||
} from "@/lib/api";
|
||||
import type { UploadProgress } from "@/lib/api";
|
||||
import type {
|
||||
BackupFile,
|
||||
IngestFile,
|
||||
@@ -56,6 +57,7 @@ function Operaciones() {
|
||||
const [confirm, setConfirm] = useState<ConfirmState>(null);
|
||||
const [confirmText, setConfirmText] = useState("");
|
||||
const [uploading, setUploading] = useState<string | null>(null);
|
||||
const [progress, setProgress] = useState<UploadProgress | null>(null);
|
||||
const [starting, setStarting] = useState(false);
|
||||
|
||||
const fileInputs = useRef<Record<string, HTMLInputElement | null>>({});
|
||||
@@ -119,14 +121,16 @@ function Operaciones() {
|
||||
setError(null);
|
||||
setNotice(null);
|
||||
setUploading(name);
|
||||
setProgress(null);
|
||||
try {
|
||||
await uploadIngest(name, file);
|
||||
await uploadIngest(name, file, setProgress);
|
||||
setNotice(`${name} cargado.`);
|
||||
refreshLists();
|
||||
} catch (e) {
|
||||
setError((e as Error)?.message ?? "No se pudo cargar el archivo.");
|
||||
} finally {
|
||||
setUploading(null);
|
||||
setProgress(null);
|
||||
const input = fileInputs.current[name];
|
||||
if (input) input.value = "";
|
||||
}
|
||||
@@ -243,45 +247,55 @@ function Operaciones() {
|
||||
</thead>
|
||||
<tbody>
|
||||
{(ingest ?? []).map((f) => (
|
||||
<tr key={f.name}>
|
||||
<td className="mono">{f.name}</td>
|
||||
<td>
|
||||
<span className={`badge ${f.present ? "badge-positive" : "badge-negative"}`}>
|
||||
{f.present ? "Presente" : "Falta"}
|
||||
</span>
|
||||
</td>
|
||||
<td className="num">{formatBytes(f.size)}</td>
|
||||
<td>{formatDateTime(f.modifiedAt)}</td>
|
||||
<td>
|
||||
<div className="row-actions">
|
||||
<input
|
||||
ref={(el) => {
|
||||
fileInputs.current[f.name] = el;
|
||||
}}
|
||||
type="file"
|
||||
style={{ display: "none" }}
|
||||
onChange={(e) => handleUpload(f.name, e.target.files?.[0])}
|
||||
/>
|
||||
<button
|
||||
className="btn btn-outline"
|
||||
type="button"
|
||||
disabled={uploading === f.name}
|
||||
onClick={() => fileInputs.current[f.name]?.click()}
|
||||
>
|
||||
{uploading === f.name ? "Cargando…" : f.present ? "Reemplazar" : "Cargar"}
|
||||
</button>
|
||||
{f.present && (
|
||||
<Fragment key={f.name}>
|
||||
<tr>
|
||||
<td className="mono">{f.name}</td>
|
||||
<td>
|
||||
<span className={`badge ${f.present ? "badge-positive" : "badge-negative"}`}>
|
||||
{f.present ? "Presente" : "Falta"}
|
||||
</span>
|
||||
</td>
|
||||
<td className="num">{formatBytes(f.size)}</td>
|
||||
<td>{formatDateTime(f.modifiedAt)}</td>
|
||||
<td>
|
||||
<div className="row-actions">
|
||||
<input
|
||||
ref={(el) => {
|
||||
fileInputs.current[f.name] = el;
|
||||
}}
|
||||
type="file"
|
||||
style={{ display: "none" }}
|
||||
onChange={(e) => handleUpload(f.name, e.target.files?.[0])}
|
||||
/>
|
||||
<button
|
||||
className="btn btn-ghost"
|
||||
className="btn btn-outline"
|
||||
type="button"
|
||||
onClick={() => handleDeleteIngest(f.name)}
|
||||
disabled={uploading === f.name}
|
||||
onClick={() => fileInputs.current[f.name]?.click()}
|
||||
>
|
||||
Eliminar
|
||||
{uploading === f.name ? "Cargando…" : f.present ? "Reemplazar" : "Cargar"}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{f.present && (
|
||||
<button
|
||||
className="btn btn-ghost"
|
||||
type="button"
|
||||
disabled={uploading === f.name}
|
||||
onClick={() => handleDeleteIngest(f.name)}
|
||||
>
|
||||
Eliminar
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{uploading === f.name && (
|
||||
<tr>
|
||||
<td colSpan={5}>
|
||||
<UploadProgressBar progress={progress} />
|
||||
</td>
|
||||
</tr>
|
||||
)}
|
||||
</Fragment>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -496,6 +510,61 @@ function Operaciones() {
|
||||
);
|
||||
}
|
||||
|
||||
/** "1:05" / "0:09" — remaining time, coarse on purpose. */
|
||||
function formatEta(seconds: number): string {
|
||||
const s = Math.max(0, Math.round(seconds));
|
||||
if (s >= 3600) {
|
||||
const h = Math.floor(s / 3600);
|
||||
const m = Math.round((s % 3600) / 60);
|
||||
return `${h} h ${m} min`;
|
||||
}
|
||||
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Live upload readout. The bar tracks bytes handed to the network; once those
|
||||
* are all sent the server still has to write the file, so the tail of the
|
||||
* upload reads "Procesando…" instead of sitting at 100%.
|
||||
*/
|
||||
function UploadProgressBar({ progress }: { progress: UploadProgress | null }) {
|
||||
const pct = progress?.fraction != null ? Math.round(progress.fraction * 100) : null;
|
||||
return (
|
||||
<div className="upload-progress">
|
||||
<div
|
||||
className={`progress-track${pct === null ? " progress-indeterminate" : ""}`}
|
||||
role="progressbar"
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-valuenow={pct ?? undefined}
|
||||
>
|
||||
<div className="progress-fill" style={{ width: `${pct ?? 100}%` }} />
|
||||
</div>
|
||||
<div className="upload-progress-stats mono">
|
||||
{progress === null ? (
|
||||
"Preparando…"
|
||||
) : progress.finishing ? (
|
||||
`Procesando en el servidor… (${formatBytes(progress.total)} enviados)`
|
||||
) : (
|
||||
<>
|
||||
{pct !== null && <strong>{pct}%</strong>}
|
||||
{progress.total > 0 && (
|
||||
<span>
|
||||
{formatBytes(progress.loaded)} / {formatBytes(progress.total)}
|
||||
</span>
|
||||
)}
|
||||
{progress.bytesPerSecond > 0 && (
|
||||
<span>{formatBytes(progress.bytesPerSecond)}/s</span>
|
||||
)}
|
||||
{progress.secondsRemaining !== null && progress.bytesPerSecond > 0 && (
|
||||
<span>faltan {formatEta(progress.secondsRemaining)}</span>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function OpTile({
|
||||
title,
|
||||
desc,
|
||||
|
||||
@@ -132,9 +132,7 @@ function BatchReview({ id }: { id: string }) {
|
||||
{error && <div className="state-box state-error">{error}</div>}
|
||||
|
||||
{processing && (
|
||||
<div className="state-box">
|
||||
Leyendo los recibos… esta pantalla se actualiza sola.
|
||||
</div>
|
||||
<ProcessingBanner docsLength={docs.length} pendingOcr={batch.byStatus.PENDING_OCR ?? 0} />
|
||||
)}
|
||||
|
||||
<SummaryCard batch={batch} readyCount={readyCount} />
|
||||
@@ -170,6 +168,55 @@ const STATUS_LABEL_BATCH: Record<string, string> = {
|
||||
FAILED: "Falló",
|
||||
};
|
||||
|
||||
/**
|
||||
* Live readout while OCR is running. The backend tells us how many pages are
|
||||
* still PENDING_OCR, so we can show real progress instead of "loading…". When
|
||||
* the docs list hasn't caught up to the upload yet (total === 0) we fall back
|
||||
* to the indeterminate bar.
|
||||
*/
|
||||
function ProcessingBanner({
|
||||
docsLength,
|
||||
pendingOcr,
|
||||
}: {
|
||||
docsLength: number;
|
||||
pendingOcr: number;
|
||||
}) {
|
||||
const done = Math.max(docsLength - pendingOcr, 0);
|
||||
const pct =
|
||||
docsLength > 0 ? Math.min(100, Math.round((done / docsLength) * 100)) : null;
|
||||
return (
|
||||
<div className="card" style={{ padding: 16 }}>
|
||||
<div className="upload-progress" style={{ padding: 0 }}>
|
||||
<div
|
||||
className={`progress-track${pct === null ? " progress-indeterminate" : ""}`}
|
||||
role="progressbar"
|
||||
aria-valuemin={0}
|
||||
aria-valuemax={100}
|
||||
aria-valuenow={pct ?? undefined}
|
||||
>
|
||||
<div className="progress-fill" style={{ width: `${pct ?? 100}%` }} />
|
||||
</div>
|
||||
<div className="upload-progress-stats">
|
||||
{pct === null ? (
|
||||
<span>Leyendo los recibos…</span>
|
||||
) : (
|
||||
<>
|
||||
<strong>{pct}%</strong>
|
||||
<span>
|
||||
{done} de {docsLength} página(s) leídas
|
||||
</span>
|
||||
{pendingOcr > 0 && <span>{pendingOcr} en cola</span>}
|
||||
</>
|
||||
)}
|
||||
<span style={{ marginLeft: "auto" }}>
|
||||
Esta pantalla se actualiza sola.
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function SummaryCard({
|
||||
batch,
|
||||
readyCount,
|
||||
|
||||
@@ -28,9 +28,15 @@ import type { ServiceKind, StatementBatch, StatementBatchStatus } from "@/lib/ty
|
||||
*/
|
||||
|
||||
/** The kinds the parsers actually recognise today. */
|
||||
const SUPPORTED: ServiceKind[] = ["ELECTRIC", "WATER", "TELEPHONE"];
|
||||
const SUPPORTED: ServiceKind[] = [
|
||||
"ELECTRIC",
|
||||
"WATER",
|
||||
"TELEPHONE",
|
||||
"GAS",
|
||||
"PROPERTY_TAX",
|
||||
];
|
||||
/** Uploadable, but every page will land in review until a parser learns it. */
|
||||
const OTHER_KINDS: ServiceKind[] = ["GAS", "PROPERTY_TAX", "FEDERAL_ZONE", "CABLE"];
|
||||
const OTHER_KINDS: ServiceKind[] = ["FEDERAL_ZONE", "CABLE"];
|
||||
|
||||
const STATUS_LABEL: Record<StatementBatchStatus, string> = {
|
||||
UPLOADED: "Recibido",
|
||||
|
||||
+98
-19
@@ -803,38 +803,117 @@ export function listIngest(): Promise<IngestFile[]> {
|
||||
return apiFetch<IngestFile[]>("/ops/ingest");
|
||||
}
|
||||
|
||||
/** Live upload stats reported to `uploadFile`'s `onProgress` callback. */
|
||||
export type UploadProgress = {
|
||||
loaded: number;
|
||||
/** 0 when the browser can't compute the request size. */
|
||||
total: number;
|
||||
/** 0..1, or null when `total` is unknown. */
|
||||
fraction: number | null;
|
||||
/** Smoothed transfer rate. */
|
||||
bytesPerSecond: number;
|
||||
/** null until a rate and a total are both known. */
|
||||
secondsRemaining: number | null;
|
||||
/** True once the bytes are sent and we're waiting on the server's reply. */
|
||||
finishing: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Multipart upload — not JSON, so it bypasses apiFetch's Content-Type. `path`
|
||||
* is API-relative (may include a query string); `filename` overrides the part
|
||||
* name sent to the server.
|
||||
* name sent to the server. Uses XHR rather than fetch because fetch has no way
|
||||
* to report request-body progress.
|
||||
*/
|
||||
export async function uploadFile(
|
||||
export function uploadFile(
|
||||
path: string,
|
||||
file: File,
|
||||
filename?: string,
|
||||
onProgress?: (p: UploadProgress) => void,
|
||||
): Promise<unknown> {
|
||||
const body = new FormData();
|
||||
body.append("file", file, filename ?? file.name);
|
||||
const res = await fetch(`${API_ORIGIN}${path}`, {
|
||||
method: "POST",
|
||||
credentials: "include",
|
||||
body,
|
||||
});
|
||||
if (!res.ok) {
|
||||
let message = `Error ${res.status}`;
|
||||
try {
|
||||
const b = await res.json();
|
||||
if (b?.message) message = b.message;
|
||||
} catch {
|
||||
/* ignore */
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open("POST", `${API_ORIGIN}${path}`);
|
||||
xhr.withCredentials = true;
|
||||
|
||||
if (onProgress) {
|
||||
// Exponentially smoothed rate — raw per-chunk deltas jump around too
|
||||
// much to read.
|
||||
let lastAt = performance.now();
|
||||
let lastLoaded = 0;
|
||||
let rate = 0;
|
||||
|
||||
xhr.upload.onprogress = (e) => {
|
||||
const now = performance.now();
|
||||
const dt = (now - lastAt) / 1000;
|
||||
if (dt >= 0.15) {
|
||||
const sample = (e.loaded - lastLoaded) / dt;
|
||||
rate = rate === 0 ? sample : rate * 0.7 + sample * 0.3;
|
||||
lastAt = now;
|
||||
lastLoaded = e.loaded;
|
||||
}
|
||||
const total = e.lengthComputable ? e.total : 0;
|
||||
onProgress({
|
||||
loaded: e.loaded,
|
||||
total,
|
||||
fraction: total ? e.loaded / total : null,
|
||||
bytesPerSecond: rate,
|
||||
secondsRemaining:
|
||||
total && rate > 0 ? (total - e.loaded) / rate : null,
|
||||
finishing: false,
|
||||
});
|
||||
};
|
||||
// Bytes are out the door; the server still has to write the file.
|
||||
xhr.upload.onload = () => {
|
||||
onProgress({
|
||||
loaded: file.size,
|
||||
total: file.size,
|
||||
fraction: 1,
|
||||
bytesPerSecond: rate,
|
||||
secondsRemaining: 0,
|
||||
finishing: true,
|
||||
});
|
||||
};
|
||||
}
|
||||
throw new ApiError(res.status, message);
|
||||
}
|
||||
return res.status === 204 ? undefined : res.json().catch(() => undefined);
|
||||
|
||||
xhr.onload = () => {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = xhr.responseText ? JSON.parse(xhr.responseText) : undefined;
|
||||
} catch {
|
||||
parsed = undefined;
|
||||
}
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
resolve(parsed);
|
||||
return;
|
||||
}
|
||||
const message =
|
||||
(parsed as { message?: string } | undefined)?.message ??
|
||||
`Error ${xhr.status}`;
|
||||
reject(new ApiError(xhr.status, message));
|
||||
};
|
||||
xhr.onerror = () =>
|
||||
reject(new ApiError(0, "Fallo de red durante la carga."));
|
||||
xhr.onabort = () => reject(new ApiError(0, "Carga cancelada."));
|
||||
xhr.ontimeout = () => reject(new ApiError(0, "Tiempo de carga agotado."));
|
||||
|
||||
xhr.send(body);
|
||||
});
|
||||
}
|
||||
|
||||
export function uploadIngest(name: string, file: File): Promise<unknown> {
|
||||
return uploadFile(`/ops/ingest/${encodeURIComponent(name)}`, file, name);
|
||||
export function uploadIngest(
|
||||
name: string,
|
||||
file: File,
|
||||
onProgress?: (p: UploadProgress) => void,
|
||||
): Promise<unknown> {
|
||||
return uploadFile(
|
||||
`/ops/ingest/${encodeURIComponent(name)}`,
|
||||
file,
|
||||
name,
|
||||
onProgress,
|
||||
);
|
||||
}
|
||||
|
||||
export function deleteIngest(name: string): Promise<unknown> {
|
||||
|
||||
@@ -47,6 +47,13 @@ ENV NODE_ENV=production
|
||||
# That breaks the pre-migrate deploy backup AND the whole "Operaciones" admin
|
||||
# panel (backup, restore, sync, re-import all shell out to these binaries).
|
||||
#
|
||||
# mdbtools-utils, NOT mdbtools. Alpine splits the project: `mdbtools` is the
|
||||
# shared library only, and the command-line tools migration/extract.py shells out
|
||||
# to (`mdb-tables`, `mdb-export`) are in the -utils subpackage. Installing the
|
||||
# wrong one builds fine and fails at run time — the re-import in the "Operaciones"
|
||||
# panel dies with:
|
||||
# RuntimeError: mdbtools not found on PATH (need mdb-tables and mdb-export)
|
||||
#
|
||||
# tesseract-ocr + tesseract-ocr-data-spa + poppler-utils drive the statement
|
||||
# OCR intake (RECEIPT_CAPTURE_SPEC §2): poppler's `pdftoppm` rasterises each
|
||||
# scanned page and tesseract reads it, with the Spanish traineddata for the
|
||||
@@ -55,7 +62,7 @@ ENV NODE_ENV=production
|
||||
# dependency. If they are absent the API still boots — the statements module
|
||||
# reports itself unavailable and only that feature is disabled — but statement
|
||||
# ingest is the point of shipping them.
|
||||
RUN apk add --no-cache python3 mdbtools mysql-client mariadb-connector-c openssl \
|
||||
RUN apk add --no-cache python3 mdbtools-utils mysql-client mariadb-connector-c openssl \
|
||||
tesseract-ocr tesseract-ocr-data-spa poppler-utils \
|
||||
&& apk add --no-cache --virtual .pybuild python3-dev build-base \
|
||||
&& rm -rf /var/cache/apk/*
|
||||
|
||||
@@ -294,10 +294,19 @@ a warning, which is what local development wants.
|
||||
Two more things the panel's dumps now do, for the same reasons the pre-migrate
|
||||
backup does them (see `deploy/scripts/pre-migrate-backup.mjs`):
|
||||
|
||||
- **`--set-gtid-purged=OFF`.** galactus is the replication *source* with GTID
|
||||
on, so without this every dump embeds `SET @@GLOBAL.GTID_PURGED` and cannot be
|
||||
restored onto the server it came from — which is precisely what the restore
|
||||
screen exists to do.
|
||||
- **`--set-gtid-purged=OFF`, but only when the dumper supports it.** galactus is
|
||||
the replication *source* with GTID on, so on a MySQL client this flag is what
|
||||
keeps every dump from embedding `SET @@GLOBAL.GTID_PURGED` and becoming
|
||||
unrestorable onto the server it came from — which is precisely what the
|
||||
restore screen exists to do. The panel, however, dumps from *inside the API
|
||||
container*, where Alpine's `mysql-client` is MariaDB's: there `mysqldump` is a
|
||||
shim over `mariadb-dump`, the flag does not exist, and passing it failed every
|
||||
backup with `mysqldump: unknown variable 'set-gtid-purged=OFF'`. So the panel
|
||||
probes `mysqldump --help` and passes the flag only if it is advertised,
|
||||
invoking `mariadb-dump` directly otherwise (MariaDB writes no GTID state
|
||||
unless asked with `--gtid`, so there is nothing to suppress). The pre-migrate
|
||||
backup keeps the flag unconditionally — it runs in a real `mysql:8.4` image,
|
||||
not in the API container.
|
||||
- **`set -o pipefail` and a `CREATE TABLE` count.** `mysqldump | gzip` reports
|
||||
gzip's exit status, and a `mysqldump` that dies on its first statement still
|
||||
produces a ~372-byte perfectly valid archive that passes `gzip -t`. Without
|
||||
|
||||
@@ -134,7 +134,8 @@ single-movement form.
|
||||
> **BUILT — 2026-08-01.** Implemented and verified end to end against real
|
||||
> scanned statements. `apps/api/src/statements/` holds the module: a swappable
|
||||
> `OcrProvider` seam with a self-hosted Tesseract implementation, per-provider
|
||||
> parsers for CFE / CESPT / Telnor, a scoped matcher, and a review queue that
|
||||
> parsers for CFE / CESPT / Telnor / gas / predial, a scoped matcher, and a
|
||||
> review queue that
|
||||
> posts through `BillingService.createBatch` with `source: "OCR"`. Web:
|
||||
> the "Captura automática (OCR)" tab of the Captura screen (upload + batch
|
||||
> list) and `/recibos/:id` (review queue with the page image beside the
|
||||
@@ -207,6 +208,54 @@ single-movement form.
|
||||
> each bill (`9`, `405`, `406`); Tesseract read `405` as `205`. Handwriting is
|
||||
> a review hint at best and is deliberately not an input to matching.
|
||||
|
||||
> **EXTENDED — gas and predial, 2026-08-01.** A second corpus (14 documents,
|
||||
> 29 pages: five municipal predial batches and ten gas invoices) added four
|
||||
> parsers — `GAS TIJUANA` plus one per municipality, because Tijuana, Rosarito
|
||||
> and Ensenada issue three completely different documents. End to end against
|
||||
> the dev database that is **21/29 auto-matched, 22/29 identified**, with the
|
||||
> provider read on 29/29 and an amount on 26/29.
|
||||
>
|
||||
> The eight review cases are all legitimate: five Tijuana pages whose municipal
|
||||
> account is not yet on file (see below), one clave not in the book, one page
|
||||
> too poorly scanned to read a clave at all, and one gas account shared by two
|
||||
> services. Excluding the structural Tijuana case, that is 21/24.
|
||||
>
|
||||
> **Five things this corpus proved:**
|
||||
>
|
||||
> 1. **Not every statement is a scan.** The gas company sends born-digital CFDI
|
||||
> invoices whose text layer is exact. Rasterising and re-recognising those
|
||||
> can only lose information — one sample turned `MEDIDOR: VM01014426` into
|
||||
> `ar (LTR): 014420` — so `OcrProvider.textPages` reads the embedded layer
|
||||
> first (`pdftotext -bbox-layout`, same poppler package as `pdftoppm`) and
|
||||
> OCR stays the fallback for real scans. Page images are still rendered and
|
||||
> stored either way, because the reviewer needs to see the paper.
|
||||
> 2. **The clave catastral is not two letters and six digits.** Positions four
|
||||
> through eight are digits in all 932 stored claves, but the third is a
|
||||
> letter in fifteen of them (`MMB01041`, `CGH52121`). Digitising the whole
|
||||
> tail maps that `B` to an `8` and produces a key matching no property.
|
||||
> 3. **Tijuana predial prints no clave catastral at all.** Its only identifier
|
||||
> is an 8-digit municipal account, carried in a 32-digit payment barcode
|
||||
> (`account(8) + DDMMYY + amount(9) + folio(9)`) that the legacy database
|
||||
> never held. It goes in `PROPERTY_TAX.meterNumber` — the same column gas
|
||||
> uses, and for the same reason: `accountNumber` holds `DATMEX.predial`,
|
||||
> which is not a per-property key and overwriting it would destroy the only
|
||||
> link back to the original records. So Tijuana pages start cold and are
|
||||
> taught by the first confirm, exactly like gas.
|
||||
> 4. **On Rosarito and Ensenada the clave is the primary key, not a fallback.**
|
||||
> Those receipts print nothing else, so a unique clave hit there is a real
|
||||
> match and auto-matches; on a utility bill that merely happens to print one
|
||||
> it stays a review hint, as before.
|
||||
> 5. **A misread `$` is the dangerous failure, not a missing one.** An Ensenada
|
||||
> receipt for `$2,203.00` OCR'd as `82,203.00` — the dollar sign read as an
|
||||
> 8, which would post a charge 37× too large and look entirely ordinary in
|
||||
> the ledger. Every predial amount therefore requires a literal `$`, and a
|
||||
> page that cannot produce one reports no amount and goes to review. Two of
|
||||
> the 29 pages take that path, which is the correct outcome for both.
|
||||
>
|
||||
> Regression cover for all of the above lives in
|
||||
> `statement-parser.spec.ts` and `tesseract.provider.spec.ts`; every fixture in
|
||||
> them is a verbatim OCR excerpt from a real receipt.
|
||||
|
||||
### Motivation (from the meeting)
|
||||
|
||||
Each utility company (CFE, water, phone, gas...) sends 300+ individual
|
||||
@@ -330,8 +379,8 @@ Per the meeting notes' own field list:
|
||||
| Agua — Número de cuenta | `WATER` | `accountNumber` | `AGUA` | ✅ populated today |
|
||||
| Zona Fed — Número de Zona Federal | `FEDERAL_ZONE` | `accountNumber` | `ZFED` | ✅ populated today |
|
||||
| Tel — Número de teléfono | `TELEPHONE` *(new)* | `accountNumber` | `Property.phone1/2/3` (currently on `Property`, not `PropertyService`) | ⚠️ schema gap — see below |
|
||||
| Impuesto — Clave Catastral | `PROPERTY_TAX` | `accountNumber` | migrated from `PREDIAL`, **not** `CLAVE` | ⚠️ needs verification — see below |
|
||||
| Gas — Número de medidor | `GAS` | `meterNumber` | not populated — folded into free-text `notes` today | ⚠️ data gap — see below |
|
||||
| Impuesto — Clave Catastral | `PROPERTY_TAX` | `Property.cadastralKey`, plus `meterNumber` for Tijuana's municipal account | `CLAVE`; `PREDIAL` is left on `accountNumber` and never matched against | ✅ built — see the 2026-08-01 extension note |
|
||||
| Gas — Número de medidor | `GAS` | `meterNumber` | not populated — folded into free-text `notes` today | ✅ 160/334 recovered from `notes` |
|
||||
|
||||
Confidence rule of thumb once a field is confirmed populated, tune after
|
||||
seeing real statements:
|
||||
|
||||
@@ -33,7 +33,7 @@ from pathlib import Path
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
from dbenv import connect, load_env
|
||||
from dbenv import connect, require, setting
|
||||
from extract import sanitize_column_name as san
|
||||
|
||||
csv.field_size_limit(300_000_000)
|
||||
@@ -102,12 +102,16 @@ def main():
|
||||
only = set(x.strip() for x in args.tables.split(",") if x.strip())
|
||||
sources = [s for s in SOURCES if not only or s["key"] in only]
|
||||
|
||||
env = load_env(args.env)
|
||||
# Process environment first, deploy/.env.<env> second, with the same
|
||||
# credential aliases the API uses — the "Operaciones" re-import runs this
|
||||
# inside the API container, which has S3_ENDPOINT / MINIO_ROOT_* injected
|
||||
# and no deploy/ directory at all.
|
||||
s3 = boto3.client(
|
||||
"s3", endpoint_url=env["S3_ENDPOINT"],
|
||||
aws_access_key_id=env["MINIO_ROOT_USER"], aws_secret_access_key=env["MINIO_ROOT_PASSWORD"],
|
||||
"s3", endpoint_url=require(args.env, "S3_ENDPOINT"),
|
||||
aws_access_key_id=require(args.env, "S3_ACCESS_KEY", "MINIO_ROOT_USER"),
|
||||
aws_secret_access_key=require(args.env, "S3_SECRET_KEY", "MINIO_ROOT_PASSWORD"),
|
||||
config=Config(signature_version="s3v4"), region_name="us-east-1")
|
||||
bucket = env["S3_BUCKET"]
|
||||
bucket = setting(args.env, "S3_BUCKET") or "jorgecuadros-documents"
|
||||
|
||||
conn = connect(args.env)
|
||||
cur = conn.cursor()
|
||||
|
||||
+31
-7
@@ -32,29 +32,53 @@ REPO = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_env(env: str) -> dict:
|
||||
"""deploy/.env.<env> parsed to a dict, or {} when the file is absent.
|
||||
|
||||
Absent is normal, not an error: the API container runs these scripts with
|
||||
DATABASE_URL / S3_* injected as real environment variables and ships no
|
||||
deploy/ directory. Use `setting()` / `require()` rather than this — they
|
||||
layer the process environment on top, which is what actually resolves."""
|
||||
f = REPO / "deploy" / f".env.{env}"
|
||||
if not f.exists():
|
||||
raise SystemExit(
|
||||
f"missing {f} — deploy the '{env}' DB stack and write its .env first "
|
||||
f"(see dbenv.py header)."
|
||||
)
|
||||
return {}
|
||||
out = {}
|
||||
for line in f.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
out[k] = v
|
||||
if "DATABASE_URL" not in out:
|
||||
raise SystemExit(f"{f} has no DATABASE_URL")
|
||||
return out
|
||||
|
||||
|
||||
def setting(env: str, *keys: str):
|
||||
"""First non-empty value for `keys`, process environment first, then
|
||||
deploy/.env.<env>. Several keys = fallback aliases (S3_ACCESS_KEY then
|
||||
MINIO_ROOT_USER, as apps/api/src/storage/storage.service.ts does)."""
|
||||
fromfile = load_env(env)
|
||||
for k in keys:
|
||||
v = os.environ.get(k) or fromfile.get(k)
|
||||
if v:
|
||||
return v
|
||||
return None
|
||||
|
||||
|
||||
def require(env: str, *keys: str) -> str:
|
||||
v = setting(env, *keys)
|
||||
if not v:
|
||||
raise SystemExit(
|
||||
f"missing {' / '.join(keys)} — set it in the environment, or deploy the "
|
||||
f"'{env}' stack and write {REPO / 'deploy' / f'.env.{env}'} "
|
||||
f"(see dbenv.py header)."
|
||||
)
|
||||
return v
|
||||
|
||||
|
||||
def database_url(env: str) -> str:
|
||||
"""Target DB URL. A DATABASE_URL in the process environment wins over
|
||||
deploy/.env.<env> — this is how the API container (which has its own
|
||||
DATABASE_URL and no deploy/.env files) drives a re-import against its own
|
||||
database."""
|
||||
return os.environ.get("DATABASE_URL") or load_env(env)["DATABASE_URL"]
|
||||
return require(env, "DATABASE_URL")
|
||||
|
||||
|
||||
def connect(env: str):
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "jorgecuadros-platform",
|
||||
"version": "1.0.2",
|
||||
"version": "1.0.5",
|
||||
"private": true,
|
||||
"workspaces": [
|
||||
"apps/*",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@jorgecuadros/database",
|
||||
"version": "1.0.2",
|
||||
"version": "1.0.5",
|
||||
"private": true,
|
||||
"main": "generated/client/index.js",
|
||||
"types": "generated/client/index.d.ts",
|
||||
|
||||
Reference in New Issue
Block a user