Compare commits

...
2 Commits
Author SHA1 Message Date
gitea-actions 127eaa9689 chore(release): v1.0.8
Build and Push Images / Build jorgecuadros-api (push) Successful in 2m14s
Build and Push Images / Build jorgecuadros-web (push) Successful in 1m42s
Cut by rmancinas via the "Cut release" workflow. Pushing the tag triggers build.yml; deploy separately with tag=1.0.8.
2026-08-03 20:47:55 +00:00
rmancinasandClaude Opus 5 7226772c22 fix(migration): recover transaction type labels and minimum balance
Two fields the customer-facing site reads were being dropped on the way in
from Access.

transform_transactions.py mapped DATOS2's type string through the Access
`TYPE OF TRX` table and stored NULL on a miss. That table is a stale
pick-list rather than a constraint — staff free-text straight into DATOS2 —
so 78 distinct values covering 3,939 rows never resolved, including
BALANCE FORWARD (1,188) and ANNUAL FEE (1,116). Nothing else on
`transactions` carries the type text, so those rows lost their label
outright and rendered blank. Now mints a type_transactions row from the
literal string when the lookup lacks it; nameEs stays NULL since only the
lookup has translations.

transform_customers.py never carried DATGRAL.TIPO, leaving
customers.minimumBalance empty on every row despite the column existing.
TIPO is the minimum-balance threshold (100/200/300/500; 1,017 of 1,172
customers carry one), not an account type as the name suggests — the
customer app shows it as `minBalance`. Added to the insert list and to the
ON DUPLICATE KEY UPDATE clause, without which --sync would silently skip
it on existing rows.

Both land on the next `run_all.py --sync` reload.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-03 12:30:03 -07:00
6 changed files with 36 additions and 7 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@jorgecuadros/api", "name": "@jorgecuadros/api",
"version": "1.0.7", "version": "1.0.8",
"private": true, "private": true,
"scripts": { "scripts": {
"build": "nest build", "build": "nest build",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@jorgecuadros/web", "name": "@jorgecuadros/web",
"version": "1.0.7", "version": "1.0.8",
"private": true, "private": true,
"scripts": { "scripts": {
"dev": "next dev -p 4500", "dev": "next dev -p 4500",
+8 -2
View File
@@ -189,6 +189,11 @@ def customer_from_utilities(row, name_index) -> dict:
customerSince=as_date(row["cliente_desde"]), customerSince=as_date(row["cliente_desde"]),
status=as_bool(row["status"]), status=as_bool(row["status"]),
feeAmount=as_decimal(row["fee"]), feeAmount=as_decimal(row["fee"]),
# DATGRAL.TIPO is the minimum-balance threshold (100/200/300/500 —
# 1,017 of 1,172 customers carry one), NOT an identification or account
# type as the column name suggests. It reaches the website as
# datosfreak.TIPO and is returned to the customer app as `minBalance`.
minimumBalance=as_decimal(row["tipo"]),
updatedAt=NOW, updatedAt=NOW,
) )
@@ -217,6 +222,7 @@ def customer_from_insurance(row, name_index) -> dict:
customerSince=None, customerSince=None,
status=1, status=1,
feeAmount=None, feeAmount=None,
minimumBalance=None,
updatedAt=NOW, updatedAt=NOW,
) )
@@ -225,7 +231,7 @@ _CUST_COLS = [
"id", "name", "nameSource", "nameMissing", "addressLine1", "addressLine2", "city", "state", "zipCode", "id", "name", "nameSource", "nameMissing", "addressLine1", "addressLine2", "city", "state", "zipCode",
"country", "phone", "mobile", "fax", "email", "notes", "identificationType", "country", "phone", "mobile", "fax", "email", "notes", "identificationType",
"identificationNumber", "identificationExpiration", "customerSince", "identificationNumber", "identificationExpiration", "customerSince",
"status", "feeAmount", "updatedAt", "status", "feeAmount", "minimumBalance", "updatedAt",
] ]
@@ -316,7 +322,7 @@ def main() -> None:
remap[rec["id"]] = stable or rec["id"] remap[rec["id"]] = stable or rec["id"]
for rec in customers: for rec in customers:
rec["id"] = remap[rec["id"]] rec["id"] = remap[rec["id"]]
cur.execute(f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE name=VALUES(name),nameSource=VALUES(nameSource),nameMissing=VALUES(nameMissing),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),city=VALUES(city),state=VALUES(state),zipCode=VALUES(zipCode),country=VALUES(country),phone=VALUES(phone),mobile=VALUES(mobile),fax=VALUES(fax),email=VALUES(email),notes=VALUES(notes),identificationType=VALUES(identificationType),identificationNumber=VALUES(identificationNumber),identificationExpiration=VALUES(identificationExpiration),customerSince=VALUES(customerSince),status=VALUES(status),feeAmount=VALUES(feeAmount),updatedAt=VALUES(updatedAt)", tuple(rec[c] for c in _CUST_COLS)) cur.execute(f"INSERT INTO customers ({','.join(f'`{c}`' for c in _CUST_COLS)}) VALUES ({placeholders}) ON DUPLICATE KEY UPDATE name=VALUES(name),nameSource=VALUES(nameSource),nameMissing=VALUES(nameMissing),addressLine1=VALUES(addressLine1),addressLine2=VALUES(addressLine2),city=VALUES(city),state=VALUES(state),zipCode=VALUES(zipCode),country=VALUES(country),phone=VALUES(phone),mobile=VALUES(mobile),fax=VALUES(fax),email=VALUES(email),notes=VALUES(notes),identificationType=VALUES(identificationType),identificationNumber=VALUES(identificationNumber),identificationExpiration=VALUES(identificationExpiration),customerSince=VALUES(customerSince),status=VALUES(status),feeAmount=VALUES(feeAmount),minimumBalance=VALUES(minimumBalance),updatedAt=VALUES(updatedAt)", tuple(rec[c] for c in _CUST_COLS))
for ref in refs: for ref in refs:
cur.execute("INSERT INTO customer_legacy_refs (id,customerId,sourceSystem,sourceTable,legacyId) VALUES (%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId)", cur.execute("INSERT INTO customer_legacy_refs (id,customerId,sourceSystem,sourceTable,legacyId) VALUES (%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId)",
(ref[0], remap[ref[1]], ref[2], ref[3], ref[4])) (ref[0], remap[ref[1]], ref[2], ref[3], ref[4]))
+24 -1
View File
@@ -112,6 +112,29 @@ def main():
type_rows.append((tid, en, s(r["espa_ol"]), 0)) type_rows.append((tid, en, s(r["espa_ol"]), 0))
type_map[en.upper()] = tid type_map[en.upper()] = tid
def type_id_for(raw) -> str | None:
"""Resolve a transaction type, minting one when the lookup lacks it.
The Access `TYPE OF TRX` table is a stale pick-list, not a constraint —
staff free-text straight into DATOS2, so 78 values covering 3,939 rows
(BALANCE FORWARD 1,188, ANNUAL FEE 1,116, IZZI 367, ...) appear in the
ledger but not the lookup. Leaving those unmapped stored typeId NULL and
lost the label outright: nothing else on `transactions` carries the type
text, so the row rendered blank and was unrecoverable after migration.
Minting from the literal keeps the display string; nameEs stays NULL
because only the lookup has translations.
"""
en = s(raw)
if not en:
return None
key = en.upper()
tid = type_map.get(key)
if tid is None:
tid = str(uuid.uuid4())
type_rows.append((tid, en, None, 0))
type_map[key] = tid
return tid
xr = load("stg_utilities", "tipo_hist") xr = load("stg_utilities", "tipo_hist")
xr_rows = [] xr_rows = []
for _, r in xr.iterrows(): for _, r in xr.iterrows():
@@ -193,7 +216,7 @@ def main():
td = dt(r["date"]) td = dt(r["date"])
if td is None: if td is None:
skip_date += 1; continue skip_date += 1; continue
tid = type_map.get((s(r["type_of_trx"]) or "").upper()) tid = type_id_for(r["type_of_trx"])
add(cid, "UTILITY", td, dec(r["chargecredit"], Decimal(0)), "MXN", add(cid, "UTILITY", td, dec(r["chargecredit"], Decimal(0)), "MXN",
period=s(r["period"]), reference=s(r["refer"]), typeid=tid, period=s(r["period"]), reference=s(r["refer"]), typeid=tid,
check=s(r["cheque"]), src_db="UTILITIES", src_tbl=legacy_tbl, check=s(r["cheque"]), src_db="UTILITIES", src_tbl=legacy_tbl,
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "jorgecuadros-platform", "name": "jorgecuadros-platform",
"version": "1.0.7", "version": "1.0.8",
"private": true, "private": true,
"workspaces": [ "workspaces": [
"apps/*", "apps/*",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@jorgecuadros/database", "name": "@jorgecuadros/database",
"version": "1.0.7", "version": "1.0.8",
"private": true, "private": true,
"main": "generated/client/index.js", "main": "generated/client/index.js",
"types": "generated/client/index.d.ts", "types": "generated/client/index.d.ts",