Add legacy database structural reference for non-Windows machines

docs/LEGACY_DATABASES.md documents all three source Access databases (every table, column, type, and known data-quality quirk) generated from a live read of the real files, so no Windows/Access driver is needed to understand their structure going forward. New migration/ tooling: catalog_schema.py connects to the real files and walks every table (including excluded scratch tables); render_catalog_md.py renders that into the doc's appendix. Raw output checked in at migration/catalog.json so the doc can be regenerated without touching Access again.
2026-07-22 01:41:53 -07:00
parent 27118f0df2
commit 7e65fc81d5
5 changed files with 12804 additions and 0 deletions
@@ -0,0 +1,108 @@
+"""
+Generates a complete structural catalog of every table in the legacy Access
+source files (including excluded/scratch tables, for documentation
+completeness) and writes it as JSON.
+
+This is the source data for docs/LEGACY_DATABASES.md. Re-run this any time
+the source .accdb/.mdb files change, then regenerate the doc from the JSON.
+
+Must run on a Windows machine with the Microsoft Access Database Engine
+ODBC driver installed (pyodbc can't reach Access files otherwise) — see
+docs/LEGACY_DATABASES.md for why non-Windows machines can't do this step
+themselves and have to work from this catalog / the staged Parquet output
+instead.
+
+Usage:
+    python catalog_schema.py --output catalog.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import extract
+from config import SOURCES
+
+
+def get_access_type_labels(cnxn, table_name: str) -> dict[str, str] | None:
+    """Best-effort: cursor.columns() gives real Access type names/sizes
+    (VARCHAR(255), DOUBLE, LONGCHAR, LONGBINARY, COUNTER, BIT, CURRENCY...)
+    but decode-errors on a subset of tables (see extract.py). Returns None
+    on failure so the caller can fall back to pandas-inferred dtypes."""
+    try:
+        cursor = cnxn.cursor()
+        labels = {}
+        for col in cursor.columns(table=table_name):
+            size = col.column_size
+            labels[col.column_name] = f"{col.type_name}({size})" if size else col.type_name
+        return labels
+    except Exception:
+        return None
+
+
+def catalog_source(source_name: str, source_cfg: dict) -> dict:
+    path = source_cfg["path"]
+    result = {"path": str(path), "tables": {}}
+
+    if not Path(path).exists():
+        result["error"] = "file not found"
+        return result
+
+    cnxn = extract.connect(path)
+    tables = extract.list_tables(cnxn)
+
+    for table_name in tables:
+        entry = {"excluded": table_name in source_cfg["exclude"]}
+        access_types = get_access_type_labels(cnxn, table_name)
+
+        try:
+            df = extract.read_table(cnxn, table_name)
+        except Exception as exc:  # noqa: BLE001
+            entry["error"] = str(exc)
+            result["tables"][table_name] = entry
+            continue
+
+        original_columns = df.attrs.get("original_columns", list(df.columns))
+        columns = []
+        for orig_name, sanitized_name in zip(original_columns, df.columns):
+            dtype = str(df[sanitized_name].dtype)
+            has_nulls = bool(df[sanitized_name].isna().any()) if len(df) else None
+            type_label = None
+            if access_types is not None:
+                type_label = access_types.get(orig_name)
+            columns.append(
+                {
+                    "name": orig_name,
+                    "access_type": type_label,
+                    "pandas_dtype": dtype,
+                    "nullable": has_nulls,
+                }
+            )
+
+        entry["row_count"] = len(df)
+        entry["column_count"] = len(columns)
+        entry["columns"] = columns
+        entry["access_type_metadata_available"] = access_types is not None
+        result["tables"][table_name] = entry
+
+    return result
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, default=Path("catalog.json"))
+    args = parser.parse_args()
+
+    catalog = {}
+    for source_name, source_cfg in SOURCES.items():
+        print(f"Cataloging {source_name}...")
+        catalog[source_name] = catalog_source(source_name, source_cfg)
+
+    args.output.write_text(json.dumps(catalog, indent=2, default=str), encoding="utf-8")
+    print(f"Wrote {args.output}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,98 @@
+"""
+Renders catalog.json (from catalog_schema.py) into a Markdown appendix:
+one section per source file, one subsection per table, with a column
+table. Excluded (scratch/template/report) tables are marked and collapsed
+to a one-liner rather than a full column dump, since they carry no data
+worth documenting column-by-column.
+
+Usage:
+    python render_catalog_md.py catalog.json --output appendix.md
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+SOURCE_TITLES = {
+    "utilities": "UTILITIES.accdb",
+    "seguros": "SEGUROS 16_be.mdb",
+    "scothia": "SCOTHIA.mdb",
+}
+
+
+def render_table(name: str, entry: dict) -> str:
+    lines = [f"#### `{name}`"]
+
+    if entry.get("excluded"):
+        lines.append("")
+        lines.append(f"**Excluded from migration** — {entry.get('row_count', '?')} rows. Not detailed here; see the exclusion rationale table above.")
+        lines.append("")
+        return "\n".join(lines)
+
+    if "error" in entry:
+        lines.append("")
+        lines.append(f"**Could not read this table:** `{entry['error']}`")
+        lines.append("")
+        return "\n".join(lines)
+
+    lines.append("")
+    lines.append(f"Rows: {entry['row_count']} | Columns: {entry['column_count']}")
+    lines.append("")
+    lines.append("| Column | Type | Nullable |")
+    lines.append("|---|---|---|")
+    for col in entry["columns"]:
+        type_label = col["access_type"] or f"_(inferred: {col['pandas_dtype']})_"
+        nullable = "yes" if col["nullable"] else ("no" if col["nullable"] is False else "—")
+        col_name = col["name"].replace("|", "\\|")
+        lines.append(f"| `{col_name}` | {type_label} | {nullable} |")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def render_source(source_name: str, source_data: dict) -> str:
+    title = SOURCE_TITLES.get(source_name, source_name)
+    lines = [f"### {title}", ""]
+
+    tables = source_data["tables"]
+    included = sorted(t for t, e in tables.items() if not e.get("excluded"))
+    excluded = sorted(t for t, e in tables.items() if e.get("excluded"))
+
+    lines.append(f"{len(tables)} tables total — {len(included)} included in migration, {len(excluded)} excluded (scratch/template/report tables).")
+    lines.append("")
+
+    for table_name in included:
+        lines.append(render_table(table_name, tables[table_name]))
+
+    if excluded:
+        lines.append("#### Excluded tables")
+        lines.append("")
+        lines.append("| Table | Rows |")
+        lines.append("|---|---|")
+        for table_name in excluded:
+            entry = tables[table_name]
+            lines.append(f"| `{table_name}` | {entry.get('row_count', '?')} |")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("catalog", type=Path)
+    parser.add_argument("--output", type=Path, default=Path("appendix.md"))
+    args = parser.parse_args()
+
+    catalog = json.loads(args.catalog.read_text(encoding="utf-8"))
+
+    sections = ["## Appendix: full table catalog", "", "Generated by `migration/catalog_schema.py` + `migration/render_catalog_md.py`. Regenerate after any change to the source files.", ""]
+    for source_name in ["utilities", "seguros", "scothia"]:
+        sections.append(render_source(source_name, catalog[source_name]))
+
+    args.output.write_text("\n".join(sections), encoding="utf-8")
+    print(f"Wrote {args.output}")
+
+
+if __name__ == "__main__":
+    main()