Add Queries/Forms/Reports reference (DAO catalog)

ODBC only exposes Tables and non-hidden SELECT queries, so this used DAO COM automation (migration/catalog_objects.py, requires pywin32) to catalog Reports, Forms, and full Query SQL text across all four files instead. Key finding: SEGUROS 16.mdb, previously noted as having zero data tables, turns out to hold all 212 Reports/149 Forms/857 Queries for the insurance line - SEGUROS 16_be.mdb is confirmed pure data storage with zero saved objects. The renewal-notice reports also reveal a RENEW/RENEW2/RENEW3 multi-notice reminder cycle not visible in the table schema. docs/LEGACY_DATABASES_OBJECTS.md documents all of this with the full 751 real queries' SQL text (business logic: billing math, year- rollover batches, duplicate/delinquency detection). Raw output at migration/objects.json. Cross-linked from RESUME.md and the existing table-only LEGACY_DATABASES.md.
2026-07-22 01:50:22 -07:00
parent 7e65fc81d5
commit 0268ed896c
7 changed files with 15303 additions and 0 deletions
@@ -0,0 +1,107 @@
+"""
+Catalogs the non-table Access objects (Queries, Forms, Reports) via DAO COM
+automation (win32com), since ODBC only exposes Tables and a subset of
+non-hidden SELECT queries (as "VIEW"). DAO exposes everything, including:
+  - QueryDefs: every saved query, with full .SQL text (this is where a lot
+    of the business logic actually lives - billing calculations, renewal
+    schedules, report data sources)
+  - Containers("Reports").Documents: every saved Report object (name only -
+    DAO's catalog interface doesn't expose RecordSource/layout; that needs
+    the full Access.Application object model, which isn't available without
+    MS Access itself installed)
+  - Containers("Forms").Documents: every saved Form object (name only, same
+    limitation)
+
+Requires: pywin32 (`pip install pywin32`) and the Access Database Engine
+(same requirement as everything else in migration/ - Windows only).
+
+Usage:
+    python catalog_objects.py --output objects.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import win32com.client
+
+from config import SOURCE_ROOT
+
+HIDDEN_QUERY_PREFIXES = ("~sq_", "~TMP")
+
+# Unlike config.SOURCES (which points "seguros" at the _be data file), the
+# Queries/Forms/Reports objects live in the linked *frontend* files - the
+# _be backend is pure data storage. Both SEGUROS files are catalogued here
+# since either could theoretically hold saved objects.
+OBJECT_SOURCES = {
+    "utilities": SOURCE_ROOT / "UTILITIES.accdb",
+    "seguros_frontend": SOURCE_ROOT / "SEGUROS 16.mdb",
+    "seguros_backend": SOURCE_ROOT / "SEGUROS 16_be.mdb",
+    "scothia": SOURCE_ROOT / "SCOTHIA.mdb",
+}
+
+
+def catalog_source(path: Path) -> dict:
+    engine = win32com.client.Dispatch("DAO.DBEngine.120")
+    db = engine.OpenDatabase(str(path))
+
+    queries = []
+    for i in range(db.QueryDefs.Count):
+        qd = db.QueryDefs(i)
+        name = qd.Name
+        is_hidden = name.startswith(HIDDEN_QUERY_PREFIXES)
+        try:
+            sql = qd.SQL
+        except Exception as exc:  # noqa: BLE001
+            sql = f"<could not read SQL: {exc}>"
+        try:
+            query_type = qd.Type  # 0=Select, 32=CrossTab, 48=Delete, 64=Update, 80=Append, 96=MakeTable, 112=DDL, 128=PassThrough, 144=SPTBulk, 160=SetOperation, 176=Procedure
+        except Exception:
+            query_type = None
+        queries.append({"name": name, "hidden": is_hidden, "sql": sql, "type": query_type})
+
+    def list_container(container_name: str) -> list[str]:
+        container = db.Containers(container_name)
+        names = []
+        for i in range(container.Documents.Count):
+            names.append(container.Documents(i).Name)
+        return sorted(names)
+
+    reports = list_container("Reports")
+    forms = list_container("Forms")
+
+    db.Close()
+
+    return {
+        "query_count": len(queries),
+        "queries": queries,
+        "report_count": len(reports),
+        "reports": reports,
+        "form_count": len(forms),
+        "forms": forms,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, default=Path("objects.json"))
+    args = parser.parse_args()
+
+    catalog = {}
+    for source_name, path in OBJECT_SOURCES.items():
+        print(f"Cataloging objects in {source_name} ({path})...")
+        if not Path(path).exists():
+            print(f"  [skip] not found")
+            continue
+        catalog[source_name] = catalog_source(path)
+        c = catalog[source_name]
+        print(f"  queries={c['query_count']} reports={c['report_count']} forms={c['form_count']}")
+
+    args.output.write_text(json.dumps(catalog, indent=2, default=str), encoding="utf-8")
+    print(f"Wrote {args.output}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,106 @@
+"""
+Renders objects.json (from catalog_objects.py) into Markdown: Reports and
+Forms as name lists (DAO's catalog interface doesn't expose RecordSource or
+layout, only names/metadata - see catalog_objects.py docstring), real
+(non-hidden) Queries with full SQL text since that's where the actual
+business logic lives, hidden ~sq_ queries collapsed to a count.
+
+Usage:
+    python render_objects_md.py objects.json --output objects.md
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+SOURCE_TITLES = {
+    "utilities": "UTILITIES.accdb",
+    "seguros_frontend": "SEGUROS 16.mdb (frontend)",
+    "seguros_backend": "SEGUROS 16_be.mdb (backend)",
+    "scothia": "SCOTHIA.mdb",
+}
+
+QUERY_TYPE_LABELS = {
+    0: "Select",
+    16: "Select (linked ODBC)",
+    32: "Crosstab",
+    48: "Delete",
+    64: "Update",
+    80: "Append",
+    96: "Make-Table",
+    112: "DDL",
+    128: "SQL Pass-Through",
+    144: "SQL Pass-Through (bulk)",
+    160: "Union",
+    176: "Procedure",
+}
+
+
+def render_name_list(title: str, names: list[str]) -> str:
+    lines = [f"##### {title} ({len(names)})", ""]
+    if not names:
+        lines.append("_none_")
+        lines.append("")
+        return "\n".join(lines)
+    for name in sorted(names):
+        lines.append(f"- `{name}`")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def render_queries(queries: list[dict]) -> str:
+    real = sorted((q for q in queries if not q["hidden"]), key=lambda q: q["name"])
+    hidden = [q for q in queries if q["hidden"]]
+
+    lines = [f"##### Queries ({len(queries)} total — {len(real)} real, {len(hidden)} hidden internal subqueries)", ""]
+    if hidden:
+        lines.append(f"_{len(hidden)} hidden `~sq_`-prefixed queries omitted — these are Access-generated internal subquery caches, not authored business logic._")
+        lines.append("")
+
+    for q in real:
+        type_label = QUERY_TYPE_LABELS.get(q["type"], f"type {q['type']}")
+        lines.append(f"###### `{q['name']}` ({type_label})")
+        lines.append("")
+        lines.append("```sql")
+        lines.append(q["sql"].strip() if q["sql"] else "-- (empty)")
+        lines.append("```")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def render_source(source_name: str, source_data: dict) -> str:
+    title = SOURCE_TITLES.get(source_name, source_name)
+    lines = [f"### {title}", ""]
+    lines.append(render_name_list("Reports", source_data["reports"]))
+    lines.append(render_name_list("Forms", source_data["forms"]))
+    lines.append(render_queries(source_data["queries"]))
+    return "\n".join(lines)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("catalog", type=Path)
+    parser.add_argument("--output", type=Path, default=Path("objects_appendix.md"))
+    args = parser.parse_args()
+
+    catalog = json.loads(args.catalog.read_text(encoding="utf-8"))
+
+    sections = [
+        "## Appendix: Reports, Forms & Queries",
+        "",
+        "Generated by `migration/catalog_objects.py` + `migration/render_objects_md.py` via DAO COM automation (not visible through plain ODBC). Reports and Forms are listed by name only — DAO's catalog interface doesn't expose RecordSource or visual layout without full MS Access installed. Queries include full SQL text, which is where the real computational business logic lives (billing math, renewal batching, filters).",
+        "",
+    ]
+    for source_name in ["utilities", "seguros_frontend", "seguros_backend", "scothia"]:
+        if source_name in catalog:
+            sections.append(render_source(source_name, catalog[source_name]))
+
+    args.output.write_text("\n".join(sections), encoding="utf-8")
+    print(f"Wrote {args.output}")
+
+
+if __name__ == "__main__":
+    main()
@@ -3,3 +3,4 @@ pandas>=2.2
 pyarrow>=15.0
 sqlalchemy>=2.0
 pymysql>=1.1
+pywin32>=306  # Windows only - needed for catalog_objects.py (DAO COM automation)