wip: ops admin panel + migration sync + crud/rbac phase-5 snapshot
Working-tree checkpoint of in-progress work carried across prior sessions on the feat/crud-rbac branch, committed so it lands on the remote alongside the CI changes. - Operaciones admin panel: apps/api/src/ops (ingest upload, backup / restore / re-import jobs) wired into app.module + RBAC abilities, and the apps/web/src/app/operaciones page. docker-compose gets INGEST_DIR / BACKUP_DIR volumes; .gitignore excludes migration/ingest + backups. - migration/sync.py plus transform_*.py / run_all / config / dbenv / blob_extract adjustments for the additive sync path. - crud/rbac phase-5 web bits: AppShell, api/labels/types libs, globals. - schema.prisma + PLAN/RESUME doc updates. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -39,6 +39,7 @@ from pathlib import Path
|
||||
import pandas as pd
|
||||
|
||||
from dbenv import connect, env_arg
|
||||
from sync import parse_mode
|
||||
|
||||
STG = Path(__file__).parent / "output"
|
||||
NULL = "∅"
|
||||
@@ -229,16 +230,16 @@ _CUST_COLS = [
|
||||
|
||||
|
||||
def main() -> None:
|
||||
env = env_arg()
|
||||
env, sync_mode = parse_mode()
|
||||
conn = connect(env)
|
||||
print(f"[customers] target env: {env}")
|
||||
cur = conn.cursor()
|
||||
|
||||
# Fresh, idempotent rebuild.
|
||||
cur.execute("SET FOREIGN_KEY_CHECKS=0")
|
||||
cur.execute("TRUNCATE TABLE customer_legacy_refs")
|
||||
cur.execute("TRUNCATE TABLE customers")
|
||||
cur.execute("SET FOREIGN_KEY_CHECKS=1")
|
||||
if not sync_mode:
|
||||
cur.execute("SET FOREIGN_KEY_CHECKS=0")
|
||||
cur.execute("TRUNCATE TABLE customer_legacy_refs")
|
||||
cur.execute("TRUNCATE TABLE customers")
|
||||
cur.execute("SET FOREIGN_KEY_CHECKS=1")
|
||||
|
||||
util = load("stg_utilities", "datgral")
|
||||
ins = load("stg_seguros", "datgral")
|
||||
@@ -291,18 +292,28 @@ def main() -> None:
|
||||
new_ins += 1
|
||||
refs.append((str(uuid.uuid4()), cust_id, "insurance", "DATGRAL", ins_id))
|
||||
|
||||
# Insert customers.
|
||||
placeholders = ",".join(["%s"] * len(_CUST_COLS))
|
||||
cur.executemany(
|
||||
f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders})",
|
||||
[tuple(rec[c] for c in _CUST_COLS) for rec in customers],
|
||||
)
|
||||
# Insert legacy refs.
|
||||
cur.executemany(
|
||||
"INSERT INTO customer_legacy_refs (id, customerId, sourceSystem, sourceTable, legacyId) "
|
||||
"VALUES (%s,%s,%s,%s,%s)",
|
||||
refs,
|
||||
)
|
||||
if sync_mode:
|
||||
existing = {}
|
||||
cur.execute("SELECT id,sourceSystem,sourceTable,legacyId,customerId FROM customer_legacy_refs")
|
||||
for rid, system, table, legacy, customer_id in cur.fetchall():
|
||||
existing[(system, table, legacy)] = (rid, customer_id)
|
||||
for rec, ref in zip(customers, refs):
|
||||
key = (ref[2], ref[3], ref[4])
|
||||
customer_id = existing.get(key, (None, rec["id"]))[1]
|
||||
rec["id"] = customer_id
|
||||
cur.execute(f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE name=VALUES(name),nameSource=VALUES(nameSource),nameMissing=VALUES(nameMissing),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),city=VALUES(city),state=VALUES(state),zipCode=VALUES(zipCode),country=VALUES(country),phone=VALUES(phone),mobile=VALUES(mobile),fax=VALUES(fax),email=VALUES(email),notes=VALUES(notes),identificationType=VALUES(identificationType),identificationNumber=VALUES(identificationNumber),identificationExpiration=VALUES(identificationExpiration),customerSince=VALUES(customerSince),status=VALUES(status),feeAmount=VALUES(feeAmount),updatedAt=VALUES(updatedAt)", tuple(rec[c] for c in _CUST_COLS))
|
||||
ref = (existing.get(key, (ref[0], customer_id))[0], customer_id, *ref[2:])
|
||||
cur.execute("INSERT INTO customer_legacy_refs (id,customerId,sourceSystem,sourceTable,legacyId) VALUES (%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId)", ref)
|
||||
else:
|
||||
cur.executemany(
|
||||
f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders})",
|
||||
[tuple(rec[c] for c in _CUST_COLS) for rec in customers],
|
||||
)
|
||||
cur.executemany(
|
||||
"INSERT INTO customer_legacy_refs (id, customerId, sourceSystem, sourceTable, legacyId) VALUES (%s,%s,%s,%s,%s)",
|
||||
refs,
|
||||
)
|
||||
|
||||
# Enrich linked customers with insurance-only ID-doc fields, and fill any
|
||||
# contact fields the utilities master left empty (COALESCE keeps master's).
|
||||
|
||||
Reference in New Issue
Block a user