diff --git a/RESUME.md b/RESUME.md index 7778b83..d4d93bb 100644 --- a/RESUME.md +++ b/RESUME.md @@ -195,9 +195,17 @@ Homebrew. No Access ODBC driver, `node_modules` not installed, staging Parquet n SCOTHIA DATOS I/E as signed amounts (income +, expense -; net +899,375.77), 66 `business_line_categories`. No customer FK; categoryId left null (concept->ramo classifier is a later enhancement). - - **NEXT:** document extraction (step 4) — pull the LONGBINARY blobs (DATMEX doc_1/2, policy - docs_1/2/foto1, DATOS docs) to object storage + write *_documents rows. Then the Customer - module API/web (Spanish-first). + - **Documents — DONE** (`migration/blob_extract.py`, migration step 4): carves embedded files + out of the Access OLE wrapper (magic-byte detection) and uploads to MinIO on cubex (stack + `jorgecuadros-dev-minio`, S3 at 192.168.4.212:9100, bucket jorgecuadros-documents), writing + service_documents/policy_documents pointer rows. Loaded 70 documents (3 service bills + + 67 policy foto/docs, 0 orphans, ~290 MB). **Data finding:** the LONGBINARY columns are + almost entirely empty — DATMEX's real bill-scan columns are ILUZ/IAGUA/IPREDIAL/ITEL (not + doc_1/doc_2), but only 3 cells populated across 1520 rows; the *_MENS tables are mail-merge + templates (correctly excluded). The 538MB/882MB source files are mostly Access bloat, not + documents. **Migration steps 1-4 COMPLETE.** + - **NEXT:** the Customer module API/web (Spanish-first) — all relational data + documents are + loaded, so build the unified customer list/search/detail view. - Full pipeline reproducible in one command: `run_all.py --env ` runs customers -> properties -> policies -> transactions -> bank in order (all idempotent); add `--stage` to re-extract from the Access files first. Verified end-to-end against dev. diff --git a/migration/blob_extract.py b/migration/blob_extract.py index 4464063..e2e099d 100644 --- a/migration/blob_extract.py +++ b/migration/blob_extract.py @@ -42,8 +42,13 @@ SOURCE_ROOT = Path.home() / "Downloads" / "JorgeCuadros-Legacy" # (key, access_file, access_table, staged_table_name, blob_cols, model) SOURCES = [ + # DATMEX's real scanned bills live in the ILUZ/IAGUA/IPREDIAL/ITEL invoice- + # image columns (per-service bill scans), not doc_1/doc_2 (which are empty). + # In practice only a handful are populated — the .accdb is mostly bloat. dict(key="datmex", file="UTILITIES.accdb", table="DATMEX", staged="DATMEX", - blobs=["doc_1", "doc_2"], model="service"), + blobs=["iluz", "iagua", "ipredial", "itel", "doc_1", "doc_2"], model="service", + doctypes={"iluz": "ELECTRIC_BILL", "iagua": "WATER_BILL", + "ipredial": "PROPERTY_TAX_BILL", "itel": "PHONE_BILL"}), dict(key="mult", file="SEGUROS 16_be.mdb", table="MULT", staged="mult", blobs=["foto1", "docs_1", "docs_2"], model="policy"), dict(key="autos_ampl", file="SEGUROS 16_be.mdb", table="TABLA AUTOS AMPL", @@ -171,7 +176,7 @@ def main(): prefix = "service" if src["model"] == "service" else "policy" key = f"{prefix}/{parent}/{src['staged']}_{ri}_{col}.{ext}" s3.put_object(Bucket=bucket, Key=key, Body=data, ContentType=ct) - dtype = col.upper() + dtype = src.get("doctypes", {}).get(col, col.upper()) if src["model"] == "service": svc_rows.append((str(uuid.uuid4()), parent, dtype, key)) else: