jorgecuadros-platform/migration/config.py

"""
Source-database manifest for the staging load (migration plan step 1).

Exclusions here are deliberately conservative: only tables that are either
(a) confirmed empty (0 rows — nothing is lost by skipping them) or (b) have
rows but are structurally not customer/business data (mail-merge document
templates, materialized Access query results) are excluded. Anything with
real rows and an ambiguous purpose (e.g. PROPANO, datosfreak, pagos email)
is loaded into staging anyway — the reconciliation pass decides what to do
with it, per the migration plan's "don't guess the rule up front" principle.
See PLAN.md (repo root) for the full rationale per table group.

Environment note: this project moved from Windows to macOS. The original
extraction path (pyodbc + the Windows Access ODBC driver in extract.py) does
not work on macOS; extraction is being reworked to use mdbtools
(`brew install mdbtools`). This SOURCE_ROOT points at the macOS location of
the four Access source files.
"""

import os
import re
from pathlib import Path

# The folder holding the four Access source files. Overridable via INGEST_DIR so
# the web "Operaciones" ingest folder (a mounted volume in the API container)
# feeds the same pipeline. Falls back to the original macOS download location
# for a plain local run.
SOURCE_ROOT = Path(
    os.environ.get("INGEST_DIR")
    or (Path.home() / "Downloads" / "JorgeCuadros-Legacy")
)

SOURCES = {
    "utilities": {
        "path": SOURCE_ROOT / "UTILITIES.accdb",
        "schema": "stg_utilities",
        # Confirmed empty (0 rows) working/scratch tables.
        "exclude": {
            "BANCO EDITOR",
            "Errores de pegado",
            "TABLE1",
            "PARA BILLING SIN",
            "PARA BILLING1",
            "PARA BILLING2",
            "PARA EDO",
            "FALTANTES AGUA",
            "FALTANTES TEL",
            "LUZ TODOS",
            "TELEFONOS FECHAS",
            "TRUSTHFEE",
            "faltantes luz",
            "billing",  # 0 rows; superseded by datos2/FEE ANUAL/fee15
            "TIT",  # 1 row, default Access "Contacts" template shell — not real data
        },
    },
    "seguros": {
        # SEGUROS 16.mdb is an empty linked front-end; all data lives in _be.
        "path": SOURCE_ROOT / "SEGUROS 16_be.mdb",
        "schema": "stg_seguros",
        "exclude": {
            # Mail-merge document templates (letters/certificates), not data.
            "AMPL MENS",
            "AMPL R MENS",
            "IN MENS",
            "LIC MENS",
            "MCA2 MENS",
            "ME MENS",
            "MF MENS",
            "RC MENSAJE",
            "RC R MENS",
            # Materialized Access query results, not source-of-truth data.
            "TODOSJC",
            "TODOS",
            "vigenta casa y auto unicos",
            # Confirmed empty scratch tables.
            "ID TABLA",
            "ID TABLATLAS",
            "TABLA LIQUIDA MCA2",
            "TABLA LIQUIDA MF",
            "TABLA LIQUIDA RES",
            "TABLA LIQUIDA TUR",
            "TABLA LIQUIDA TUR ENDOSO",
            "TABLA AUTOS LIMIT R",
            "BORRA",
            "GENERICO_OLD",
            "TIT",
        },
    },
    "scothia": {
        "path": SOURCE_ROOT / "SCOTHIA.mdb",
        "schema": "stg_scothia",
        "exclude": {
            "INFORME",
            "INFORME BA",
            "FECHAIF",  # date-range UI parameter table, not data
        },
    },
}


# --- prior-period archives ----------------------------------------------
#
# Legacy ran a year-end *corte*: it summed the closing year, wrote that total
# back as each customer's Jan-1 BALANCE FORWARD, and started the next year
# clean. Access keeps the closed year as a whole-database snapshot named for
# the period it holds — `2025.accdb` is UTILITIES as it stood when 2025 was
# cut — and the office archives one per year.
#
# Only the ledger is staged out of a snapshot. Everything else in it (DATMEX,
# PROFILE, EFECTIVO, ...) is a year-stale copy of a table the live
# UTILITIES.accdb already provides, and staging all ~50 of them would triple
# the extract time to import data we would then have to ignore. DATGRAL comes
# along solely to check that a NUMid still means the same customer it did that
# year; see the recycle guard in transform_transactions.py.
#
# The cash side is deliberately NOT taken from the snapshot: `EFECTIVO` is a
# lifetime journal, so the snapshot's copy is a subset of the live one and
# importing it would double-book every prior-year receipt.
PERIOD_FILE_RE = re.compile(r"^(\d{4})\.accdb$", re.IGNORECASE)
PERIOD_TABLES = {"datos2", "DATGRAL"}


def period_schema(year: int) -> str:
    return f"stg_period_{year}"


def discover_periods(root: Path) -> dict[str, dict]:
    """Find every `YYYY.accdb` archive sitting in the ingest folder.

    Discovery is by filename because that is the whole upload contract: the
    operator drops `2025.accdb` on the Operaciones page and the period is 2025.
    Nothing inside the file names the year — a snapshot's `datos2` looks
    identical to the live one — so the name is the only declaration of intent
    we get, and it is what the allowlist on the upload endpoint enforces.
    """
    found: dict[str, dict] = {}
    if not root.is_dir():
        return found
    for path in sorted(root.iterdir()):
        m = PERIOD_FILE_RE.match(path.name)
        if not m:
            continue
        year = int(m.group(1))
        found[f"period_{year}"] = {
            "path": path,
            "schema": period_schema(year),
            "exclude": set(),
            "include": set(PERIOD_TABLES),
            "period_year": year,
        }
    return found


SOURCES.update(discover_periods(SOURCE_ROOT))