rmancinasandClaude Opus 4.8
f1ef1c70b3
Working-tree checkpoint of in-progress work carried across prior sessions on the feat/crud-rbac branch, committed so it lands on the remote alongside the CI changes. - Operaciones admin panel: apps/api/src/ops (ingest upload, backup / restore / re-import jobs) wired into app.module + RBAC abilities, and the apps/web/src/app/operaciones page. docker-compose gets INGEST_DIR / BACKUP_DIR volumes; .gitignore excludes migration/ingest + backups. - migration/sync.py plus transform_*.py / run_all / config / dbenv / blob_extract adjustments for the additive sync path. - crud/rbac phase-5 web bits: AppShell, api/labels/types libs, globals. - schema.prisma + PLAN/RESUME doc updates. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
98 lines
3.4 KiB
Python
98 lines
3.4 KiB
Python
"""
|
|
Source-database manifest for the staging load (migration plan step 1).
|
|
|
|
Exclusions here are deliberately conservative: only tables that are either
|
|
(a) confirmed empty (0 rows — nothing is lost by skipping them) or (b) have
|
|
rows but are structurally not customer/business data (mail-merge document
|
|
templates, materialized Access query results) are excluded. Anything with
|
|
real rows and an ambiguous purpose (e.g. PROPANO, datosfreak, pagos email)
|
|
is loaded into staging anyway — the reconciliation pass decides what to do
|
|
with it, per the migration plan's "don't guess the rule up front" principle.
|
|
See PLAN.md (repo root) for the full rationale per table group.
|
|
|
|
Environment note: this project moved from Windows to macOS. The original
|
|
extraction path (pyodbc + the Windows Access ODBC driver in extract.py) does
|
|
not work on macOS; extraction is being reworked to use mdbtools
|
|
(`brew install mdbtools`). This SOURCE_ROOT points at the macOS location of
|
|
the four Access source files.
|
|
"""
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
# The folder holding the four Access source files. Overridable via INGEST_DIR so
|
|
# the web "Operaciones" ingest folder (a mounted volume in the API container)
|
|
# feeds the same pipeline. Falls back to the original macOS download location
|
|
# for a plain local run.
|
|
SOURCE_ROOT = Path(
|
|
os.environ.get("INGEST_DIR")
|
|
or (Path.home() / "Downloads" / "JorgeCuadros-Legacy")
|
|
)
|
|
|
|
SOURCES = {
|
|
"utilities": {
|
|
"path": SOURCE_ROOT / "UTILITIES.accdb",
|
|
"schema": "stg_utilities",
|
|
# Confirmed empty (0 rows) working/scratch tables.
|
|
"exclude": {
|
|
"BANCO EDITOR",
|
|
"Errores de pegado",
|
|
"TABLE1",
|
|
"PARA BILLING SIN",
|
|
"PARA BILLING1",
|
|
"PARA BILLING2",
|
|
"PARA EDO",
|
|
"FALTANTES AGUA",
|
|
"FALTANTES TEL",
|
|
"LUZ TODOS",
|
|
"TELEFONOS FECHAS",
|
|
"TRUSTHFEE",
|
|
"faltantes luz",
|
|
"billing", # 0 rows; superseded by datos2/FEE ANUAL/fee15
|
|
"TIT", # 1 row, default Access "Contacts" template shell — not real data
|
|
},
|
|
},
|
|
"seguros": {
|
|
# SEGUROS 16.mdb is an empty linked front-end; all data lives in _be.
|
|
"path": SOURCE_ROOT / "SEGUROS 16_be.mdb",
|
|
"schema": "stg_seguros",
|
|
"exclude": {
|
|
# Mail-merge document templates (letters/certificates), not data.
|
|
"AMPL MENS",
|
|
"AMPL R MENS",
|
|
"IN MENS",
|
|
"LIC MENS",
|
|
"MCA2 MENS",
|
|
"ME MENS",
|
|
"MF MENS",
|
|
"RC MENSAJE",
|
|
"RC R MENS",
|
|
# Materialized Access query results, not source-of-truth data.
|
|
"TODOSJC",
|
|
"TODOS",
|
|
"vigenta casa y auto unicos",
|
|
# Confirmed empty scratch tables.
|
|
"ID TABLA",
|
|
"ID TABLATLAS",
|
|
"TABLA LIQUIDA MCA2",
|
|
"TABLA LIQUIDA MF",
|
|
"TABLA LIQUIDA RES",
|
|
"TABLA LIQUIDA TUR",
|
|
"TABLA LIQUIDA TUR ENDOSO",
|
|
"TABLA AUTOS LIMIT R",
|
|
"BORRA",
|
|
"GENERICO_OLD",
|
|
"TIT",
|
|
},
|
|
},
|
|
"scothia": {
|
|
"path": SOURCE_ROOT / "SCOTHIA.mdb",
|
|
"schema": "stg_scothia",
|
|
"exclude": {
|
|
"INFORME",
|
|
"INFORME BA",
|
|
"FECHAIF", # date-range UI parameter table, not data
|
|
},
|
|
},
|
|
}
|