wip: ops admin panel + migration sync + crud/rbac phase-5 snapshot

Working-tree checkpoint of in-progress work carried across prior
sessions on the feat/crud-rbac branch, committed so it lands on the
remote alongside the CI changes.

- Operaciones admin panel: apps/api/src/ops (ingest upload, backup /
  restore / re-import jobs) wired into app.module + RBAC abilities, and
  the apps/web/src/app/operaciones page. docker-compose gets INGEST_DIR
  / BACKUP_DIR volumes; .gitignore excludes migration/ingest + backups.
- migration/sync.py plus transform_*.py / run_all / config / dbenv /
  blob_extract adjustments for the additive sync path.
- crud/rbac phase-5 web bits: AppShell, api/labels/types libs, globals.
- schema.prisma + PLAN/RESUME doc updates.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

migration/blob_extract.py

@@ -38,7 +38,9 @@ from extract import sanitize_column_name as san
 
 csv.field_size_limit(300_000_000)
 
-SOURCE_ROOT = Path.home() / "Downloads" / "JorgeCuadros-Legacy"
+# Same source folder as the rest of the pipeline (config.SOURCE_ROOT honours
+# INGEST_DIR — the web "Operaciones" ingest volume).
+from config import SOURCE_ROOT
 
 # (key, access_file, access_table, staged_table_name, blob_cols, model)
 SOURCES = [

migration/config.py

@@ -17,9 +17,17 @@ not work on macOS; extraction is being reworked to use mdbtools
 the four Access source files.
 """
 
+import os
 from pathlib import Path
 
-SOURCE_ROOT = Path.home() / "Downloads" / "JorgeCuadros-Legacy"
+# The folder holding the four Access source files. Overridable via INGEST_DIR so
+# the web "Operaciones" ingest folder (a mounted volume in the API container)
+# feeds the same pipeline. Falls back to the original macOS download location
+# for a plain local run.
+SOURCE_ROOT = Path(
+    os.environ.get("INGEST_DIR")
+    or (Path.home() / "Downloads" / "JorgeCuadros-Legacy")
+)
 
 SOURCES = {
     "utilities": {

migration/dbenv.py

@@ -22,6 +22,7 @@ Usage in a script:
 from __future__ import annotations
 
 import argparse
+import os
 from pathlib import Path
 from urllib.parse import unquote, urlparse
 
@@ -48,8 +49,16 @@ def load_env(env: str) -> dict:
     return out
 
 
+def database_url(env: str) -> str:
+    """Target DB URL. A DATABASE_URL in the process environment wins over
+    deploy/.env.<env> — this is how the API container (which has its own
+    DATABASE_URL and no deploy/.env files) drives a re-import against its own
+    database."""
+    return os.environ.get("DATABASE_URL") or load_env(env)["DATABASE_URL"]
+
+
 def connect(env: str):
-    url = load_env(env)["DATABASE_URL"]
+    url = database_url(env)
     u = urlparse(url)  # mysql://user:pass@host:port/db
     return pymysql.connect(
         host=u.hostname,

migration/run_all.py

@@ -39,13 +39,21 @@ PY = sys.executable  # the venv python running this orchestrator
 # service_documents / policy_documents, which would otherwise leave the
 # uploaded MinIO objects with no rows pointing at them.
 STEPS = [
-    "transform_customers.py",     # customers + customer_legacy_refs (everything FKs to these)
-    "transform_properties.py",    # properties + services + trust accounts
-    "transform_policies.py",      # policies + installments/vehicles/drivers/benef/claims/adjusters
-    "transform_transactions.py",  # shared ledger + type_transactions + exchange_rates
-    "prune_empty_customers.py",   # drop customers with no property/policy/transaction
-    "transform_bank.py",          # SCOTHIA bank register (no customer FK; independent)
-    "blob_extract.py",            # document pointers; must follow properties + policies
+    "transform_customers.py",
+    "transform_properties.py",
+    "transform_policies.py",
+    "transform_transactions.py",
+    "prune_empty_customers.py",
+    "transform_bank.py",
+    "blob_extract.py",
+]
+
+SYNC_STEPS = [
+    "transform_customers.py",
+    "transform_properties.py",
+    "transform_policies.py",
+    "transform_transactions.py",
+    "transform_bank.py",
 ]
 
 
@@ -61,13 +69,18 @@ def main() -> None:
     ap.add_argument("--env", default="dev", help="target environment (reads deploy/.env.<env>)")
     ap.add_argument("--stage", action="store_true",
                     help="re-run the raw staging load first (needs the Access files + mdbtools)")
+    ap.add_argument("--sync", action="store_true",
+                    help="upsert legacy rows and archive removed legacy rows; preserve manual rows")
     args = ap.parse_args()
 
     if args.stage:
         run([PY, str(HERE / "load_staging.py"), "--output-dir", str(HERE / "output")])
 
-    for step in STEPS:
-        run([PY, str(HERE / step), "--env", args.env])
+    for step in SYNC_STEPS if args.sync else STEPS:
+        cmd = [PY, str(HERE / step), "--env", args.env]
+        if args.sync:
+            cmd.append("--sync")
+        run(cmd)
 
     print(f"\n✓ migration complete for env={args.env}")
 

migration/sync.py

@@ -0,0 +1,27 @@
+"""Shared CLI and SQL helpers for migration modes."""
+
+from __future__ import annotations
+
+import argparse
+
+
+def parse_mode() -> tuple[str, bool]:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--env", default="dev")
+    parser.add_argument("--sync", action="store_true")
+    args = parser.parse_args()
+    return args.env, args.sync
+
+
+def existing_ids(cursor, table: str, key_columns: tuple[str, ...], where: str = "") -> dict[tuple, str]:
+    columns = ",".join(("id", *key_columns))
+    cursor.execute(f"SELECT {columns} FROM {table} {where}")
+    return {tuple(row[1:]): row[0] for row in cursor.fetchall()}
+
+
+def delete_missing(cursor, table: str, key_columns: tuple[str, ...], seen: set[tuple], where: str) -> int:
+    rows = existing_ids(cursor, table, key_columns, where)
+    stale = [row_id for key, row_id in rows.items() if key not in seen]
+    if stale:
+        cursor.executemany(f"DELETE FROM {table} WHERE id=%s", [(row_id,) for row_id in stale])
+    return len(stale)

migration/transform_bank.py

@@ -28,6 +28,7 @@ from pathlib import Path
 import pandas as pd
 
 from dbenv import connect, env_arg
+from sync import parse_mode
 
 STG = Path(__file__).parent / "output" / "stg_scothia"
 NULL = "∅"
@@ -72,7 +73,7 @@ def load(name):
 
 
 def main():
-    env = env_arg()
+    env, sync_mode = parse_mode()
     conn = connect(env)
     print(f"[bank] target env: {env}")
     c = conn.cursor()
@@ -109,16 +110,17 @@ def main():
     for _, r in load("datos_e").iterrows():
         add(r, -(dec(r["egreso"], Decimal(0))), income=False)
 
-    c.execute("SET FOREIGN_KEY_CHECKS=0")
-    for t in ("bank_transactions", "business_line_categories"):
-        c.execute(f"TRUNCATE TABLE {t}")
-    c.execute("SET FOREIGN_KEY_CHECKS=1")
-
-    c.executemany("INSERT INTO business_line_categories (id,name) VALUES (%s,%s)", cats)
-    c.executemany(
-        "INSERT INTO bank_transactions (id,transactionDate,transactionType,reference,concept,"
-        "amount,categoryId,cleared,transferred,notes,amountInWords,legacySourceTable,legacyId) "
-        "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", rows)
+    if sync_mode:
+        for row in rows:
+            c.execute("INSERT INTO bank_transactions (id,transactionDate,transactionType,reference,concept,amount,categoryId,cleared,transferred,notes,amountInWords,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE transactionDate=VALUES(transactionDate),transactionType=VALUES(transactionType),reference=VALUES(reference),concept=VALUES(concept),amount=VALUES(amount),cleared=VALUES(cleared),transferred=VALUES(transferred),notes=VALUES(notes),amountInWords=VALUES(amountInWords),voidedAt=NULL", row)
+    else:
+        c.execute("SET FOREIGN_KEY_CHECKS=0")
+        for t in ("bank_transactions", "business_line_categories"):
+            c.execute(f"TRUNCATE TABLE {t}")
+        c.execute("SET FOREIGN_KEY_CHECKS=1")
+        c.executemany("INSERT INTO business_line_categories (id,name) VALUES (%s,%s)", cats)
+        c.executemany(
+            "INSERT INTO bank_transactions (id,transactionDate,transactionType,reference,concept,amount,categoryId,cleared,transferred,notes,amountInWords,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", rows)
     conn.commit()
 
     def count(t):

migration/transform_customers.py

@@ -39,6 +39,7 @@ from pathlib import Path
 import pandas as pd
 
 from dbenv import connect, env_arg
+from sync import parse_mode
 
 STG = Path(__file__).parent / "output"
 NULL = "∅"
@@ -229,16 +230,16 @@ _CUST_COLS = [
 
 
 def main() -> None:
-    env = env_arg()
+    env, sync_mode = parse_mode()
     conn = connect(env)
     print(f"[customers] target env: {env}")
     cur = conn.cursor()
 
-    # Fresh, idempotent rebuild.
-    cur.execute("SET FOREIGN_KEY_CHECKS=0")
-    cur.execute("TRUNCATE TABLE customer_legacy_refs")
-    cur.execute("TRUNCATE TABLE customers")
-    cur.execute("SET FOREIGN_KEY_CHECKS=1")
+    if not sync_mode:
+        cur.execute("SET FOREIGN_KEY_CHECKS=0")
+        cur.execute("TRUNCATE TABLE customer_legacy_refs")
+        cur.execute("TRUNCATE TABLE customers")
+        cur.execute("SET FOREIGN_KEY_CHECKS=1")
 
     util = load("stg_utilities", "datgral")
     ins = load("stg_seguros", "datgral")
@@ -291,18 +292,28 @@ def main() -> None:
             new_ins += 1
         refs.append((str(uuid.uuid4()), cust_id, "insurance", "DATGRAL", ins_id))
 
-    # Insert customers.
     placeholders = ",".join(["%s"] * len(_CUST_COLS))
-    cur.executemany(
-        f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders})",
-        [tuple(rec[c] for c in _CUST_COLS) for rec in customers],
-    )
-    # Insert legacy refs.
-    cur.executemany(
-        "INSERT INTO customer_legacy_refs (id, customerId, sourceSystem, sourceTable, legacyId) "
-        "VALUES (%s,%s,%s,%s,%s)",
-        refs,
-    )
+    if sync_mode:
+        existing = {}
+        cur.execute("SELECT id,sourceSystem,sourceTable,legacyId,customerId FROM customer_legacy_refs")
+        for rid, system, table, legacy, customer_id in cur.fetchall():
+            existing[(system, table, legacy)] = (rid, customer_id)
+        for rec, ref in zip(customers, refs):
+            key = (ref[2], ref[3], ref[4])
+            customer_id = existing.get(key, (None, rec["id"]))[1]
+            rec["id"] = customer_id
+            cur.execute(f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE name=VALUES(name),nameSource=VALUES(nameSource),nameMissing=VALUES(nameMissing),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),city=VALUES(city),state=VALUES(state),zipCode=VALUES(zipCode),country=VALUES(country),phone=VALUES(phone),mobile=VALUES(mobile),fax=VALUES(fax),email=VALUES(email),notes=VALUES(notes),identificationType=VALUES(identificationType),identificationNumber=VALUES(identificationNumber),identificationExpiration=VALUES(identificationExpiration),customerSince=VALUES(customerSince),status=VALUES(status),feeAmount=VALUES(feeAmount),updatedAt=VALUES(updatedAt)", tuple(rec[c] for c in _CUST_COLS))
+            ref = (existing.get(key, (ref[0], customer_id))[0], customer_id, *ref[2:])
+            cur.execute("INSERT INTO customer_legacy_refs (id,customerId,sourceSystem,sourceTable,legacyId) VALUES (%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId)", ref)
+    else:
+        cur.executemany(
+            f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders})",
+            [tuple(rec[c] for c in _CUST_COLS) for rec in customers],
+        )
+        cur.executemany(
+            "INSERT INTO customer_legacy_refs (id, customerId, sourceSystem, sourceTable, legacyId) VALUES (%s,%s,%s,%s,%s)",
+            refs,
+        )
 
     # Enrich linked customers with insurance-only ID-doc fields, and fill any
     # contact fields the utilities master left empty (COALESCE keeps master's).

migration/transform_policies.py

@@ -37,6 +37,7 @@ from pathlib import Path
 import pandas as pd
 
 from dbenv import connect, env_arg
+from sync import parse_mode
 
 STG = Path(__file__).parent / "output" / "stg_seguros"
 LEGACY_DB = "SEGUROS 16_be"
@@ -171,7 +172,7 @@ def load(name):
 
 
 def main():
-    env = env_arg()
+    env, sync_mode = parse_mode()
     conn = connect(env)
     print(f"[policies] target env: {env}")
     c = conn.cursor()
@@ -315,35 +316,44 @@ def main():
                        dt(r["fecha_cheque"]), s(r["num_cheque"]),
                        1 if truthy(r["concluido"]) else 0, s(r["resolucion"])))
 
-    # --- write (children first on truncate) ---
-    c.execute("SET FOREIGN_KEY_CHECKS=0")
-    for t in ("policy_payment_installments", "insured_drivers", "policy_beneficiaries",
-              "claims", "vehicles", "policy_documents", "policies", "policy_types",
-              "insurance_providers", "adjusters"):
-        c.execute(f"TRUNCATE TABLE {t}")
-    c.execute("SET FOREIGN_KEY_CHECKS=1")
+    if sync_mode:
+        c.execute("SELECT id,name FROM policy_types")
+        ptype_ids = dict(c.fetchall())
+        for n in ptypes:
+            ptype_ids.setdefault(n, str(uuid.uuid4()))
+        c.executemany("INSERT INTO policy_types (id,name) VALUES (%s,%s) ON DUPLICATE KEY UPDATE name=VALUES(name)", [(i, n) for n, i in ptype_ids.items()])
+        c.execute("SELECT id,name FROM insurance_providers")
+        prov_ids = dict(c.fetchall())
+        for n in providers:
+            prov_ids.setdefault(n, str(uuid.uuid4()))
+        c.executemany("INSERT INTO insurance_providers (id,name) VALUES (%s,%s) ON DUPLICATE KEY UPDATE name=VALUES(name)", [(i, n) for n, i in prov_ids.items()])
+        for p in policies:
+            p = list(p); p[3] = ptype_ids[p[3]]; p[4] = prov_ids.get(p[4])
+            c.execute(f"INSERT INTO policies ({pol_cols}) VALUES ({','.join(['%s'] * 23)}) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),policyNumber=VALUES(policyNumber),policyTypeId=VALUES(policyTypeId),insuranceProviderId=VALUES(insuranceProviderId),agentName=VALUES(agentName),policyDate=VALUES(policyDate),policyFrom=VALUES(policyFrom),policyTo=VALUES(policyTo),netPremium=VALUES(netPremium),policyFee=VALUES(policyFee),commission=VALUES(commission),total=VALUES(total),currency=VALUES(currency),observations=VALUES(observations),coveragesJson=VALUES(coveragesJson),liquidated=VALUES(liquidated),liquidationNumber=VALUES(liquidationNumber),liquidationDate=VALUES(liquidationDate),updatedAt=VALUES(updatedAt),archivedAt=NULL", tuple(p))
+    else:
+        ptype_ids = {n: str(uuid.uuid4()) for n in ptypes}
+        c.executemany("INSERT INTO policy_types (id,name) VALUES (%s,%s)", [(i, n) for n, i in ptype_ids.items()])
+        prov_ids = {n: str(uuid.uuid4()) for n in providers}
+        c.executemany("INSERT INTO insurance_providers (id,name) VALUES (%s,%s)", [(i, n) for n, i in prov_ids.items()])
+        c.executemany("INSERT INTO adjusters (id,company,city,name,phone,beeper) VALUES (%s,%s,%s,%s,%s,%s)", adj_rows)
 
-    ptype_ids = {n: str(uuid.uuid4()) for n in ptypes}
-    c.executemany("INSERT INTO policy_types (id,name) VALUES (%s,%s)",
-                  [(i, n) for n, i in ptype_ids.items()])
-    prov_ids = {n: str(uuid.uuid4()) for n in providers}
-    c.executemany("INSERT INTO insurance_providers (id,name) VALUES (%s,%s)",
-                  [(i, n) for n, i in prov_ids.items()])
-    c.executemany("INSERT INTO adjusters (id,company,city,name,phone,beeper) VALUES (%s,%s,%s,%s,%s,%s)", adj_rows)
-
-    # patch policyType/provider FKs into policy tuples
     pol_cols = ("id,policyNumber,customerId,policyTypeId,insuranceProviderId,agentName,policyDate,"
                 "policyFrom,policyTo,netPremium,policyFee,commission,total,currency,observations,"
                 "coveragesJson,liquidated,liquidationNumber,liquidationDate,legacySourceDb,"
                 "legacySourceTable,legacyId,updatedAt")
-    fixed = []
-    for p in policies:
-        p = list(p)
-        p[3] = ptype_ids.get(p[3])          # ptype name -> policyTypeId
-        p[4] = prov_ids.get(p[4])           # provider name -> insuranceProviderId
-        fixed.append(tuple(p))
-    ph = ",".join(["%s"] * 23)
-    c.executemany(f"INSERT INTO policies ({pol_cols}) VALUES ({ph})", fixed)
+    if not sync_mode:
+        fixed = []
+        for p in policies:
+            p = list(p)
+            p[3] = ptype_ids.get(p[3])
+            p[4] = prov_ids.get(p[4])
+            fixed.append(tuple(p))
+        ph = ",".join(["%s"] * 23)
+        c.executemany(f"INSERT INTO policies ({pol_cols}) VALUES ({ph})", fixed)
+    else:
+        c.executemany("INSERT INTO policies (id,policyNumber,customerId,policyTypeId,insuranceProviderId,agentName,policyDate,policyFrom,policyTo,netPremium,policyFee,commission,total,currency,observations,coveragesJson,liquidated,liquidationNumber,liquidationDate,legacySourceDb,legacySourceTable,legacyId,updatedAt) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),policyNumber=VALUES(policyNumber),policyTypeId=VALUES(policyTypeId),insuranceProviderId=VALUES(insuranceProviderId),agentName=VALUES(agentName),policyDate=VALUES(policyDate),policyFrom=VALUES(policyFrom),policyTo=VALUES(policyTo),netPremium=VALUES(netPremium),policyFee=VALUES(policyFee),commission=VALUES(commission),total=VALUES(total),currency=VALUES(currency),observations=VALUES(observations),coveragesJson=VALUES(coveragesJson),liquidated=VALUES(liquidated),liquidationNumber=VALUES(liquidationNumber),liquidationDate=VALUES(liquidationDate),updatedAt=VALUES(updatedAt),archivedAt=NULL", [tuple([p[0],p[1],p[2],ptype_ids.get(p[3]),prov_ids.get(p[4]),*p[5:]]) for p in policies])
+
+
 
     c.executemany("INSERT INTO policy_payment_installments "
                   "(id,policyId,sequence,amount,currency,paidDate,checkNumber,isCash) "

migration/transform_properties.py

@@ -34,6 +34,7 @@ from pathlib import Path
 import pandas as pd
 
 from dbenv import connect, env_arg
+from sync import delete_missing, existing_ids, parse_mode
 
 STG = Path(__file__).parent / "output" / "stg_utilities"
 NULL = "∅"
@@ -100,8 +101,8 @@ def jkey(numer, casa, direc):
     ])
 
 
-def main() -> None:
-    env = env_arg()
+def main():
+    env, sync_mode = parse_mode()
     conn = connect(env)
     print(f"[properties] target env: {env}")
     cur = conn.cursor()
@@ -119,6 +120,7 @@ def main() -> None:
         flags[jkey(r["numerid"], r["casa"], r["direccion"])] = r
 
     props, services, trusts = [], [], []
+    prop_keys: set[tuple] = set()
     skipped_no_customer = 0
     matched_profile = 0
 
@@ -129,6 +131,8 @@ def main() -> None:
             skipped_no_customer += 1
             continue
 
+        legacy_id = str(int(row["_row_num"]))
+        prop_keys.add(("DATMEX", legacy_id))
         pid = str(uuid.uuid4())
         addr2_parts = []
         for lbl, col in (("CASA", "casa"), ("MZ", "manzana"), ("LOTE", "lote")):
@@ -137,7 +141,8 @@ def main() -> None:
         props.append((
             pid, cust_id, s(row["direccion"]), ", ".join(addr2_parts) or None,
             s(row["telefono"]), s(row["telefono2"]), s(row["telefono3"]),
-            s(row["zona"]), "DATMEX", nid,
+             s(row["zona"]), "DATMEX", legacy_id,
+
         ))
 
         pfrow = flags.get(jkey(row["numer_id"], row["casa"], row["direccion"]))
@@ -199,21 +204,28 @@ def main() -> None:
             trusts.append((str(uuid.uuid4()), pid, bank, s(row["trust_num"]),
                            as_dec(row["bfee"]), as_date(row["vence1"]), as_date(row["vence2"])))
 
-    # Fresh rebuild (children first).
-    cur.execute("SET FOREIGN_KEY_CHECKS=0")
-    for t in ("property_services", "service_documents", "trust_accounts", "properties"):
-        cur.execute(f"TRUNCATE TABLE {t}")
-    cur.execute("SET FOREIGN_KEY_CHECKS=1")
-
-    cur.executemany(
-        "INSERT INTO properties (id,customerId,addressLine1,addressLine2,phone1,phone2,"
-        "phone3,zone,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", props)
-    cur.executemany(
-        "INSERT INTO property_services (id,propertyId,kind,accountNumber,meterNumber,route,"
-        "dueDay,active,notes) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", services)
-    cur.executemany(
-        "INSERT INTO trust_accounts (id,propertyId,bankName,trustNumber,bankFee,dueDate1,dueDate2)"
-        " VALUES (%s,%s,%s,%s,%s,%s,%s)", trusts)
+    # Fresh rebuild (children first), or additive upsert for legacy-owned rows.
+    if sync_mode:
+        existing = existing_ids(cur, "properties", ("legacySourceTable", "legacyId"), "WHERE legacyId IS NOT NULL")
+        for row in props:
+            key = (row[8], row[9])
+            if key in existing:
+                row = list(row); row[0] = existing[key]
+            cur.execute(
+                "INSERT INTO properties (id,customerId,addressLine1,addressLine2,phone1,phone2,phone3,zone,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),phone1=VALUES(phone1),phone2=VALUES(phone2),phone3=VALUES(phone3),zone=VALUES(zone),archivedAt=NULL", tuple(row))
+        delete_missing(cur, "properties", ("legacySourceTable", "legacyId"), prop_keys, "WHERE legacyId IS NOT NULL")
+        cur.execute("DELETE ps FROM property_services ps JOIN properties p ON p.id=ps.propertyId WHERE p.legacyId IS NOT NULL")
+        cur.execute("DELETE ta FROM trust_accounts ta JOIN properties p ON p.id=ta.propertyId WHERE p.legacyId IS NOT NULL")
+        cur.executemany("INSERT INTO property_services (id,propertyId,kind,accountNumber,meterNumber,route,dueDay,active,notes) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", services)
+        cur.executemany("INSERT INTO trust_accounts (id,propertyId,bankName,trustNumber,bankFee,dueDate1,dueDate2) VALUES (%s,%s,%s,%s,%s,%s,%s)", trusts)
+    else:
+        cur.execute("SET FOREIGN_KEY_CHECKS=0")
+        for t in ("property_services", "service_documents", "trust_accounts", "properties"):
+            cur.execute(f"TRUNCATE TABLE {t}")
+        cur.execute("SET FOREIGN_KEY_CHECKS=1")
+        cur.executemany("INSERT INTO properties (id,customerId,addressLine1,addressLine2,phone1,phone2,phone3,zone,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", props)
+        cur.executemany("INSERT INTO property_services (id,propertyId,kind,accountNumber,meterNumber,route,dueDay,active,notes) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", services)
+        cur.executemany("INSERT INTO trust_accounts (id,propertyId,bankName,trustNumber,bankFee,dueDate1,dueDate2) VALUES (%s,%s,%s,%s,%s,%s,%s)", trusts)
     conn.commit()
 
     cur.execute("SELECT COUNT(*) FROM properties"); n_p = cur.fetchone()[0]

migration/transform_transactions.py

@@ -35,6 +35,7 @@ from pathlib import Path
 import pandas as pd
 
 from dbenv import connect, env_arg
+from sync import parse_mode
 
 STG = Path(__file__).parent / "output"
 NULL = "∅"
@@ -90,7 +91,7 @@ def load(src, name):
 
 
 def main():
-    env = env_arg()
+    env, sync_mode = parse_mode()
     conn = connect(env)
     print(f"[transactions] target env: {env}")
     c = conn.cursor()
@@ -225,18 +226,17 @@ def main():
     iva()
     efectivo_like("stg_seguros", "efectivo", "INSURANCE", ins_cust, "SEGUROS 16_be", "EFECTIVO")
 
-    # --- write ---
-    c.execute("SET FOREIGN_KEY_CHECKS=0")
-    for t in ("transactions", "type_transactions", "exchange_rates"):
-        c.execute(f"TRUNCATE TABLE {t}")
-    c.execute("SET FOREIGN_KEY_CHECKS=1")
-
-    c.executemany("INSERT INTO type_transactions (id,nameEn,nameEs,isService) VALUES (%s,%s,%s,%s)", type_rows)
-    c.executemany("INSERT INTO exchange_rates (id,rate,effectiveDate,effectiveHour) VALUES (%s,%s,%s,%s)", xr_rows)
-    c.executemany(
-        "INSERT INTO transactions (id,customerId,domain,typeId,transactionDate,period,reference,"
-        "amount,currency,exchangeRate,checkNumber,message,outstanding,legacySourceDb,"
-        "legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", tx)
+    if sync_mode:
+        c.executemany("INSERT INTO transactions (id,customerId,domain,typeId,transactionDate,period,reference,amount,currency,exchangeRate,checkNumber,message,outstanding,legacySourceDb,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),domain=VALUES(domain),typeId=VALUES(typeId),transactionDate=VALUES(transactionDate),period=VALUES(period),reference=VALUES(reference),amount=VALUES(amount),currency=VALUES(currency),checkNumber=VALUES(checkNumber),message=VALUES(message),updatedAt=NOW(),voidedAt=NULL", tx)
+    else:
+        c.execute("SET FOREIGN_KEY_CHECKS=0")
+        for t in ("transactions", "type_transactions", "exchange_rates"):
+            c.execute(f"TRUNCATE TABLE {t}")
+        c.execute("SET FOREIGN_KEY_CHECKS=1")
+        c.executemany("INSERT INTO type_transactions (id,nameEn,nameEs,isService) VALUES (%s,%s,%s,%s)", type_rows)
+        c.executemany("INSERT INTO exchange_rates (id,rate,effectiveDate,effectiveHour) VALUES (%s,%s,%s,%s)", xr_rows)
+        c.executemany(
+            "INSERT INTO transactions (id,customerId,domain,typeId,transactionDate,period,reference,amount,currency,exchangeRate,checkNumber,message,outstanding,legacySourceDb,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", tx)
     conn.commit()
 
     def count(t):