Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3b02c6944f | ||
|
|
683fd37b08 | ||
|
|
14c6183aa2 | ||
|
|
5352d49ecf | ||
|
|
2169ffa78d | ||
|
|
17d83291c3 | ||
|
|
6a97242fc3 | ||
|
|
7981c715ce | ||
|
|
d173c9e9a0 | ||
|
|
e9a5ee9e90 |
@@ -96,7 +96,14 @@ NEXT_PUBLIC_API_ORIGIN=http://localhost:3001
|
|||||||
```
|
```
|
||||||
|
|
||||||
The API loads `DATABASE_URL`, `SESSION_SECRET`, `WEB_ORIGIN`, and optional
|
The API loads `DATABASE_URL`, `SESSION_SECRET`, `WEB_ORIGIN`, and optional
|
||||||
`PORT` (default `3001`). The web app only needs `NEXT_PUBLIC_API_ORIGIN`.
|
`PORT` (default `3001`). `WEB_ORIGIN` is comma-separated — list every origin the
|
||||||
|
app is reached under, or credentialed fetches from the missing ones fail CORS.
|
||||||
|
|
||||||
|
The web app needs no API URL of its own: the browser derives it from the page it
|
||||||
|
loaded (same host on port `3001` over plain HTTP, or the same-origin `/api` path
|
||||||
|
behind a TLS proxy). Set `NEXT_PUBLIC_API_ORIGIN` (dev) or `API_ORIGIN` (deploy,
|
||||||
|
read at request time) only to override that — for instance when running the API
|
||||||
|
on a non-default port.
|
||||||
|
|
||||||
### 3. Start MySQL
|
### 3. Start MySQL
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@jorgecuadros/api",
|
"name": "@jorgecuadros/api",
|
||||||
"version": "1.0.14",
|
"version": "1.0.17",
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "nest build",
|
"build": "nest build",
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ export type Ability =
|
|||||||
| "customer:create"
|
| "customer:create"
|
||||||
| "customer:update"
|
| "customer:update"
|
||||||
| "customer:delete"
|
| "customer:delete"
|
||||||
|
| "customer:portal-access"
|
||||||
| "policy:create"
|
| "policy:create"
|
||||||
| "policy:update"
|
| "policy:update"
|
||||||
| "policy:delete"
|
| "policy:delete"
|
||||||
@@ -48,6 +49,12 @@ export const ABILITY_MIN: Record<Ability, Role> = {
|
|||||||
"customer:create": "STAFF",
|
"customer:create": "STAFF",
|
||||||
"customer:update": "STAFF",
|
"customer:update": "STAFF",
|
||||||
"customer:delete": "ADMIN",
|
"customer:delete": "ADMIN",
|
||||||
|
// Assigning a portal NUMid is granting someone the ability to log in to
|
||||||
|
// my.jorgecuadros.com and read an account, so it sits above customer:update:
|
||||||
|
// editing a phone number is the day job, handing out portal identity is not.
|
||||||
|
// It is also close to irreversible in practice — the id is what the customer
|
||||||
|
// then types at every login.
|
||||||
|
"customer:portal-access": "MANAGER",
|
||||||
"policy:create": "STAFF",
|
"policy:create": "STAFF",
|
||||||
"policy:update": "STAFF",
|
"policy:update": "STAFF",
|
||||||
"policy:delete": "MANAGER",
|
"policy:delete": "MANAGER",
|
||||||
|
|||||||
@@ -0,0 +1,180 @@
|
|||||||
|
import { Prisma } from "@jorgecuadros/database";
|
||||||
|
import {
|
||||||
|
BALANCE_FLOOR_JOIN,
|
||||||
|
BALANCE_FORWARD_TYPE,
|
||||||
|
BillingService,
|
||||||
|
NOT_SUPERSEDED,
|
||||||
|
} from "./billing.service";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The balance floor drops rows a later BALANCE FORWARD already accounts for.
|
||||||
|
*
|
||||||
|
* It is worth testing because it fails silently: nothing throws, the numbers are
|
||||||
|
* just wrong, and they were wrong for years — the whole book read +20.6M MXN in
|
||||||
|
* credit because every customer's pre-cutover history was counted twice, once
|
||||||
|
* inside their opening balance and once as itself.
|
||||||
|
*/
|
||||||
|
describe("balance floor", () => {
|
||||||
|
describe("SQL fragments", () => {
|
||||||
|
it("binds the type name rather than interpolating it", () => {
|
||||||
|
// A literal would be a second place to edit if the label ever changes,
|
||||||
|
// and this string reaches SQL from a module constant.
|
||||||
|
expect(BALANCE_FLOOR_JOIN.values).toEqual([BALANCE_FORWARD_TYPE]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keys the floor to the row's own customer", () => {
|
||||||
|
// Without this the derived table cross-joins and every customer inherits
|
||||||
|
// the earliest BALANCE FORWARD in the book.
|
||||||
|
expect(BALANCE_FLOOR_JOIN.sql).toContain(
|
||||||
|
"bfloor ON bfloor.customerId = t.customerId",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("takes the most recent opening balance, not the first", () => {
|
||||||
|
// A customer accumulates one BALANCE FORWARD per year. MIN would floor at
|
||||||
|
// the oldest and leave every intervening year double-counted.
|
||||||
|
expect(BALANCE_FLOOR_JOIN.sql).toContain("MAX(bf.transactionDate)");
|
||||||
|
expect(BALANCE_FLOOR_JOIN.sql).not.toContain("MIN(bf.transactionDate)");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores voided opening balances when locating the floor", () => {
|
||||||
|
expect(BALANCE_FLOOR_JOIN.sql).toContain("bf.voidedAt IS NULL");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is inclusive of the opening balance row itself", () => {
|
||||||
|
// `>` instead of `>=` would drop the carried balance and understate every
|
||||||
|
// customer by exactly that amount.
|
||||||
|
expect(NOT_SUPERSEDED.sql).toContain("t.transactionDate >= bfloor.floorDate");
|
||||||
|
expect(NOT_SUPERSEDED.sql).not.toMatch(/transactionDate\s*>\s*bfloor/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves customers with no opening balance untouched", () => {
|
||||||
|
// NULL comparisons are never true, so without the explicit IS NULL branch
|
||||||
|
// a customer who has no BALANCE FORWARD row loses their entire ledger.
|
||||||
|
expect(NOT_SUPERSEDED.sql).toContain("bfloor.floorDate IS NULL");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("only ever references the alias the join defines", () => {
|
||||||
|
// The predicate is useless without the join; pairing them wrongly is a
|
||||||
|
// runtime "unknown column", so keep the alias identical in both.
|
||||||
|
const aliases = NOT_SUPERSEDED.sql.match(/bfloor\.\w+/g) ?? [];
|
||||||
|
expect(aliases.length).toBeGreaterThan(0);
|
||||||
|
for (const ref of aliases) {
|
||||||
|
expect(BALANCE_FLOOR_JOIN.sql).toContain(ref.split(".")[1]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("statement()", () => {
|
||||||
|
/**
|
||||||
|
* One customer means one floor date, so the statement uses a scalar lookup
|
||||||
|
* instead of the join. Asserting on the `where` Prisma is handed is the only
|
||||||
|
* way to see it without a database.
|
||||||
|
*/
|
||||||
|
function serviceWith(floor: Date | null) {
|
||||||
|
const findMany = jest.fn().mockResolvedValue([]);
|
||||||
|
const prisma = {
|
||||||
|
customer: {
|
||||||
|
findUnique: jest.fn().mockResolvedValue({
|
||||||
|
id: "c1",
|
||||||
|
name: "CUADROS, JORGE H.",
|
||||||
|
preferredCurrency: "USD",
|
||||||
|
_count: { properties: 0, policies: 0 },
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
transaction: {
|
||||||
|
findFirst: jest
|
||||||
|
.fn()
|
||||||
|
.mockResolvedValue(floor ? { transactionDate: floor } : null),
|
||||||
|
findMany,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
service: new BillingService(prisma as never),
|
||||||
|
prisma,
|
||||||
|
findMany,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
it("looks the floor up from the customer's newest opening balance", async () => {
|
||||||
|
const { service, prisma } = serviceWith(new Date("2026-01-01T00:00:00Z"));
|
||||||
|
|
||||||
|
await service.statement("c1");
|
||||||
|
|
||||||
|
expect(prisma.transaction.findFirst).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
where: {
|
||||||
|
customerId: "c1",
|
||||||
|
voidedAt: null,
|
||||||
|
type: { nameEn: BALANCE_FORWARD_TYPE },
|
||||||
|
},
|
||||||
|
orderBy: { transactionDate: "desc" },
|
||||||
|
select: { transactionDate: true },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("bounds the statement at the floor, inclusive", async () => {
|
||||||
|
const floor = new Date("2026-01-01T00:00:00Z");
|
||||||
|
const { service, findMany } = serviceWith(floor);
|
||||||
|
|
||||||
|
await service.statement("c1");
|
||||||
|
|
||||||
|
expect(findMany.mock.calls[0][0].where).toMatchObject({
|
||||||
|
customerId: "c1",
|
||||||
|
transactionDate: { gte: floor },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("applies no date bound when the customer has no opening balance", async () => {
|
||||||
|
const { service, findMany } = serviceWith(null);
|
||||||
|
|
||||||
|
await service.statement("c1");
|
||||||
|
|
||||||
|
expect(findMany.mock.calls[0][0].where).not.toHaveProperty(
|
||||||
|
"transactionDate",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the source-table exclusion alongside the floor", async () => {
|
||||||
|
// The two guards answer different questions — one reproduces legacy's
|
||||||
|
// DATOS2-only materialization, the other drops superseded history — and
|
||||||
|
// dropping either one changes the customer's balance.
|
||||||
|
const { service, findMany } = serviceWith(new Date("2026-01-01T00:00:00Z"));
|
||||||
|
|
||||||
|
await service.statement("c1");
|
||||||
|
|
||||||
|
const where = findMany.mock.calls[0][0].where;
|
||||||
|
expect(where.OR).toEqual([
|
||||||
|
{ legacySourceTable: null },
|
||||||
|
{ legacySourceTable: { notIn: expect.arrayContaining(["EFECTIVO"]) } },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("regression: NUMid 501", () => {
|
||||||
|
/**
|
||||||
|
* The arithmetic that exposed the bug, pinned so it cannot silently return.
|
||||||
|
* Figures measured against the live ledger on 2026-08-05.
|
||||||
|
*/
|
||||||
|
const openingBalance = new Prisma.Decimal("-6732.29");
|
||||||
|
const activitySinceOpening = new Prisma.Decimal("-7333.00");
|
||||||
|
const preCutoverCashAlreadyInOpening = new Prisma.Decimal("3596.00");
|
||||||
|
|
||||||
|
it("matches the legacy portal once superseded rows are dropped", () => {
|
||||||
|
expect(openingBalance.plus(activitySinceOpening).toFixed(2)).toBe(
|
||||||
|
"-14065.29",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reproduces the wrong figure when they are not", () => {
|
||||||
|
expect(
|
||||||
|
openingBalance
|
||||||
|
.plus(activitySinceOpening)
|
||||||
|
.plus(preCutoverCashAlreadyInOpening)
|
||||||
|
.toFixed(2),
|
||||||
|
).toBe("-10469.29");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -104,24 +104,31 @@ interface BalanceRow {
|
|||||||
nameMissing: number;
|
nameMissing: number;
|
||||||
city: string | null;
|
city: string | null;
|
||||||
state: string | null;
|
state: string | null;
|
||||||
movements: bigint | number | string;
|
movements: RawCount;
|
||||||
balanceMxn: Prisma.Decimal | null;
|
balanceMxn: Prisma.Decimal | null;
|
||||||
balanceUsd: Prisma.Decimal | null;
|
balanceUsd: Prisma.Decimal | null;
|
||||||
chargesMxn: Prisma.Decimal | null;
|
chargesMxn: Prisma.Decimal | null;
|
||||||
creditsMxn: Prisma.Decimal | null;
|
creditsMxn: Prisma.Decimal | null;
|
||||||
chargesUsd: Prisma.Decimal | null;
|
chargesUsd: Prisma.Decimal | null;
|
||||||
creditsUsd: Prisma.Decimal | null;
|
creditsUsd: Prisma.Decimal | null;
|
||||||
utilityMovements: bigint | number | string;
|
utilityMovements: RawCount;
|
||||||
insuranceMovements: bigint | number | string;
|
insuranceMovements: RawCount;
|
||||||
lastMovement: Date | null;
|
lastMovement: Date | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Raw-query counts come back in three shapes depending on the aggregate:
|
* Every shape a raw-query count can arrive in. `COUNT(*)` is a bigint,
|
||||||
* `COUNT(*)` as bigint, `SUM(bool)` as a decimal *string*, and plain numbers.
|
* `SUM(bool)` is a Prisma.Decimal, and plain numbers occur too — none of which
|
||||||
* Normalize all of them before they reach the client as JSON.
|
* survive JSON serialization the way the client expects.
|
||||||
*/
|
*/
|
||||||
function num(v: bigint | number | string | null | undefined): number {
|
type RawCount = bigint | number | string | Prisma.Decimal;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalizes a raw-query count before it reaches the client as JSON. A bigint
|
||||||
|
* throws on JSON.stringify and a Decimal serializes to a *string*, so counts
|
||||||
|
* must not be passed through untouched.
|
||||||
|
*/
|
||||||
|
function num(v: RawCount | null | undefined): number {
|
||||||
if (v === null || v === undefined) return 0;
|
if (v === null || v === undefined) return 0;
|
||||||
return typeof v === "number" ? v : Number(v);
|
return typeof v === "number" ? v : Number(v);
|
||||||
}
|
}
|
||||||
@@ -152,6 +159,56 @@ const NOT_VOIDED: Prisma.TransactionWhereInput = { voidedAt: null };
|
|||||||
*/
|
*/
|
||||||
const NOT_OUTSTANDING: Prisma.TransactionWhereInput = { outstanding: false };
|
const NOT_OUTSTANDING: Prisma.TransactionWhereInput = { outstanding: false };
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The legacy type name for a carried-forward opening balance.
|
||||||
|
*
|
||||||
|
* These rows are not movements. Access materialized one per customer per year,
|
||||||
|
* dated Jan 1, holding the closing balance of everything before it — that is
|
||||||
|
* what let the portal keep each year in its own table (`datosfreak` = current,
|
||||||
|
* `2025`, `2024`, ...) and still show a correct running balance from a single
|
||||||
|
* year's rows.
|
||||||
|
*/
|
||||||
|
export const BALANCE_FORWARD_TYPE = "BALANCE FORWARD";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-customer date of the most recent BALANCE FORWARD row.
|
||||||
|
*
|
||||||
|
* Joined rather than correlated: one small derived table (1,170 rows) beats a
|
||||||
|
* subquery evaluated per ledger row.
|
||||||
|
*/
|
||||||
|
export const BALANCE_FLOOR_JOIN = Prisma.sql`
|
||||||
|
LEFT JOIN (
|
||||||
|
SELECT bf.customerId, MAX(bf.transactionDate) AS floorDate
|
||||||
|
FROM transactions bf
|
||||||
|
JOIN type_transactions bft ON bft.id = bf.typeId
|
||||||
|
WHERE bft.nameEn = ${BALANCE_FORWARD_TYPE} AND bf.voidedAt IS NULL
|
||||||
|
GROUP BY bf.customerId
|
||||||
|
) bfloor ON bfloor.customerId = t.customerId`;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Excludes rows a later BALANCE FORWARD already accounts for.
|
||||||
|
*
|
||||||
|
* WHY THIS EXISTS. The platform holds both the synthetic BALANCE FORWARD rows
|
||||||
|
* and the real pre-cutover history they summarize, so summing a customer's
|
||||||
|
* whole ledger counts that history twice — once inside the opening balance,
|
||||||
|
* once as itself. NUMid 501 read -10,469.29 on the worklist against -14,065.29
|
||||||
|
* on the customer's own statement and on the legacy portal, the gap being two
|
||||||
|
* cash receipts from 2009 and 2012 that the 2026 opening balance had already
|
||||||
|
* absorbed.
|
||||||
|
*
|
||||||
|
* The scale is what settles it: summed the old way the entire book came to
|
||||||
|
* +20,605,447.86 MXN — the office owing its customers 20.6 million pesos.
|
||||||
|
* Floored, it is -56,855.90, a modest net receivable. A receivables ledger
|
||||||
|
* cannot be 20M in credit.
|
||||||
|
*
|
||||||
|
* Applies to BALANCES ONLY, in the same spirit as NOT_OUTSTANDING: the movement
|
||||||
|
* browser still totals every captured row, because "how much water did we
|
||||||
|
* capture in April" is a question about what was recorded, not about what is
|
||||||
|
* owed. Customers with no BALANCE FORWARD row (the floor is NULL) are
|
||||||
|
* unaffected.
|
||||||
|
*/
|
||||||
|
export const NOT_SUPERSEDED = Prisma.sql`(bfloor.floorDate IS NULL OR t.transactionDate >= bfloor.floorDate)`;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Source tables excluded from the customer-facing statement.
|
* Source tables excluded from the customer-facing statement.
|
||||||
*
|
*
|
||||||
@@ -402,22 +459,24 @@ export class BillingService {
|
|||||||
MAX(t.transactionDate) AS lastMovement
|
MAX(t.transactionDate) AS lastMovement
|
||||||
FROM customers c
|
FROM customers c
|
||||||
JOIN transactions t ON t.customerId = c.id
|
JOIN transactions t ON t.customerId = c.id
|
||||||
WHERE t.voidedAt IS NULL AND t.outstanding = 0 ${nameFilter} ${txFilter}
|
${BALANCE_FLOOR_JOIN}
|
||||||
|
WHERE t.voidedAt IS NULL AND t.outstanding = 0 AND ${NOT_SUPERSEDED} ${nameFilter} ${txFilter}
|
||||||
GROUP BY c.id, c.name, c.nameSource, c.nameMissing, c.city, c.state
|
GROUP BY c.id, c.name, c.nameSource, c.nameMissing, c.city, c.state
|
||||||
${having}
|
${having}
|
||||||
${orderBy}
|
${orderBy}
|
||||||
LIMIT ${pageSize} OFFSET ${(page - 1) * pageSize}
|
LIMIT ${pageSize} OFFSET ${(page - 1) * pageSize}
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const counted = await this.prisma.$queryRaw<{ total: bigint | number | string }[]>`
|
const counted = await this.prisma.$queryRaw<{ total: RawCount }[]>`
|
||||||
SELECT COUNT(*) AS total FROM (
|
SELECT COUNT(*) AS total FROM (
|
||||||
SELECT c.id
|
SELECT c.id
|
||||||
FROM customers c
|
FROM customers c
|
||||||
JOIN transactions t ON t.customerId = c.id
|
JOIN transactions t ON t.customerId = c.id
|
||||||
|
${BALANCE_FLOOR_JOIN}
|
||||||
-- Must match the page query's filters exactly, or the total disagrees
|
-- Must match the page query's filters exactly, or the total disagrees
|
||||||
-- with the rows. (The void exclusion was missing here before the
|
-- with the rows. (The void exclusion was missing here before the
|
||||||
-- outstanding work; a voided-only customer inflated the count.)
|
-- outstanding work; a voided-only customer inflated the count.)
|
||||||
WHERE t.voidedAt IS NULL AND t.outstanding = 0 ${nameFilter} ${txFilter}
|
WHERE t.voidedAt IS NULL AND t.outstanding = 0 AND ${NOT_SUPERSEDED} ${nameFilter} ${txFilter}
|
||||||
GROUP BY c.id
|
GROUP BY c.id
|
||||||
${having}
|
${having}
|
||||||
) x
|
) x
|
||||||
@@ -458,9 +517,18 @@ export class BillingService {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Top-line figures for the billing page header. */
|
/**
|
||||||
|
* Top-line figures for the billing page header.
|
||||||
|
*
|
||||||
|
* Two different questions live here and they use different row sets.
|
||||||
|
* `movements`, `ledgerCustomers`, `crossLineCustomers` and the date range are
|
||||||
|
* INVENTORY — what is stored — and count everything not voided. Everything
|
||||||
|
* under `byCurrency` / `byDomain` is a BALANCE, so it applies NOT_SUPERSEDED
|
||||||
|
* and drops rows an opening balance already accounts for. The four aggregates
|
||||||
|
* moved from Prisma groupBy to raw SQL to express that join; groupBy cannot.
|
||||||
|
*/
|
||||||
async stats() {
|
async stats() {
|
||||||
const [movements, ledgerCustomers, byCurrency, byDomain] = await Promise.all([
|
const [movements, ledgerCustomers] = await Promise.all([
|
||||||
this.prisma.transaction.count({ where: NOT_VOIDED }),
|
this.prisma.transaction.count({ where: NOT_VOIDED }),
|
||||||
this.prisma.transaction
|
this.prisma.transaction
|
||||||
.findMany({
|
.findMany({
|
||||||
@@ -469,34 +537,47 @@ export class BillingService {
|
|||||||
select: { customerId: true },
|
select: { customerId: true },
|
||||||
})
|
})
|
||||||
.then((r) => r.length),
|
.then((r) => r.length),
|
||||||
this.prisma.transaction.groupBy({
|
|
||||||
by: ["currency"],
|
|
||||||
where: NOT_VOIDED,
|
|
||||||
_sum: { amount: true },
|
|
||||||
_count: { _all: true },
|
|
||||||
}),
|
|
||||||
this.prisma.transaction.groupBy({
|
|
||||||
by: ["domain", "currency"],
|
|
||||||
where: NOT_VOIDED,
|
|
||||||
_sum: { amount: true },
|
|
||||||
_count: { _all: true },
|
|
||||||
}),
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const charges = await this.prisma.transaction.groupBy({
|
const byCurrency = await this.prisma.$queryRaw<
|
||||||
by: ["currency"],
|
{
|
||||||
where: { AND: [{ amount: { lt: 0 } }, NOT_VOIDED] },
|
currency: string;
|
||||||
_sum: { amount: true },
|
net: Prisma.Decimal | null;
|
||||||
_count: { _all: true },
|
count: RawCount;
|
||||||
});
|
charges: Prisma.Decimal | null;
|
||||||
const credits = await this.prisma.transaction.groupBy({
|
chargeCount: RawCount;
|
||||||
by: ["currency"],
|
credits: Prisma.Decimal | null;
|
||||||
where: { AND: [{ amount: { gt: 0 } }, NOT_VOIDED] },
|
creditCount: RawCount;
|
||||||
_sum: { amount: true },
|
}[]
|
||||||
_count: { _all: true },
|
>`
|
||||||
});
|
SELECT t.currency AS currency,
|
||||||
const chargeMap = new Map(charges.map((c) => [c.currency, c]));
|
SUM(t.amount) AS net,
|
||||||
const creditMap = new Map(credits.map((c) => [c.currency, c]));
|
COUNT(*) AS count,
|
||||||
|
SUM(CASE WHEN t.amount < 0 THEN t.amount ELSE 0 END) AS charges,
|
||||||
|
SUM(t.amount < 0) AS chargeCount,
|
||||||
|
SUM(CASE WHEN t.amount > 0 THEN t.amount ELSE 0 END) AS credits,
|
||||||
|
SUM(t.amount > 0) AS creditCount
|
||||||
|
FROM transactions t
|
||||||
|
${BALANCE_FLOOR_JOIN}
|
||||||
|
WHERE t.voidedAt IS NULL AND ${NOT_SUPERSEDED}
|
||||||
|
GROUP BY t.currency
|
||||||
|
`;
|
||||||
|
|
||||||
|
const byDomain = await this.prisma.$queryRaw<
|
||||||
|
{
|
||||||
|
domain: string;
|
||||||
|
currency: string;
|
||||||
|
net: Prisma.Decimal | null;
|
||||||
|
count: RawCount;
|
||||||
|
}[]
|
||||||
|
>`
|
||||||
|
SELECT t.domain AS domain, t.currency AS currency,
|
||||||
|
SUM(t.amount) AS net, COUNT(*) AS count
|
||||||
|
FROM transactions t
|
||||||
|
${BALANCE_FLOOR_JOIN}
|
||||||
|
WHERE t.voidedAt IS NULL AND ${NOT_SUPERSEDED}
|
||||||
|
GROUP BY t.domain, t.currency
|
||||||
|
`;
|
||||||
|
|
||||||
// How many customers sit on each side of the line, per currency — the
|
// How many customers sit on each side of the line, per currency — the
|
||||||
// headline for a receivables view. Counted in SQL; a customer can be
|
// headline for a receivables view. Counted in SQL; a customer can be
|
||||||
@@ -504,16 +585,19 @@ export class BillingService {
|
|||||||
const sides = await this.prisma.$queryRaw<
|
const sides = await this.prisma.$queryRaw<
|
||||||
{
|
{
|
||||||
currency: string;
|
currency: string;
|
||||||
owing: bigint | number | string;
|
owing: RawCount;
|
||||||
inCredit: bigint | number | string;
|
inCredit: RawCount;
|
||||||
}[]
|
}[]
|
||||||
>`
|
>`
|
||||||
SELECT currency,
|
SELECT currency,
|
||||||
SUM(bal < -0.005) AS owing,
|
SUM(bal < -0.005) AS owing,
|
||||||
SUM(bal > 0.005) AS inCredit
|
SUM(bal > 0.005) AS inCredit
|
||||||
FROM (
|
FROM (
|
||||||
SELECT customerId, currency, SUM(amount) AS bal
|
SELECT t.customerId, t.currency, SUM(t.amount) AS bal
|
||||||
FROM transactions WHERE voidedAt IS NULL GROUP BY customerId, currency
|
FROM transactions t
|
||||||
|
${BALANCE_FLOOR_JOIN}
|
||||||
|
WHERE t.voidedAt IS NULL AND ${NOT_SUPERSEDED}
|
||||||
|
GROUP BY t.customerId, t.currency
|
||||||
) x
|
) x
|
||||||
GROUP BY currency
|
GROUP BY currency
|
||||||
`;
|
`;
|
||||||
@@ -534,7 +618,7 @@ export class BillingService {
|
|||||||
|
|
||||||
// Customers whose ledger spans both business lines — the whole reason this
|
// Customers whose ledger spans both business lines — the whole reason this
|
||||||
// module is one view instead of two.
|
// module is one view instead of two.
|
||||||
const crossLine = await this.prisma.$queryRaw<{ n: bigint | number | string }[]>`
|
const crossLine = await this.prisma.$queryRaw<{ n: RawCount }[]>`
|
||||||
SELECT COUNT(*) AS n FROM (
|
SELECT COUNT(*) AS n FROM (
|
||||||
SELECT customerId FROM transactions WHERE voidedAt IS NULL
|
SELECT customerId FROM transactions WHERE voidedAt IS NULL
|
||||||
GROUP BY customerId HAVING COUNT(DISTINCT domain) > 1
|
GROUP BY customerId HAVING COUNT(DISTINCT domain) > 1
|
||||||
@@ -549,20 +633,20 @@ export class BillingService {
|
|||||||
lastMovement: lastRow?.transactionDate ?? null,
|
lastMovement: lastRow?.transactionDate ?? null,
|
||||||
byCurrency: byCurrency.map((c) => ({
|
byCurrency: byCurrency.map((c) => ({
|
||||||
currency: c.currency,
|
currency: c.currency,
|
||||||
net: c._sum.amount,
|
net: c.net,
|
||||||
count: c._count._all,
|
count: num(c.count),
|
||||||
charges: chargeMap.get(c.currency)?._sum.amount ?? null,
|
charges: c.charges,
|
||||||
chargeCount: chargeMap.get(c.currency)?._count._all ?? 0,
|
chargeCount: num(c.chargeCount),
|
||||||
credits: creditMap.get(c.currency)?._sum.amount ?? null,
|
credits: c.credits,
|
||||||
creditCount: creditMap.get(c.currency)?._count._all ?? 0,
|
creditCount: num(c.creditCount),
|
||||||
owing: num(sideMap.get(c.currency)?.owing),
|
owing: num(sideMap.get(c.currency)?.owing),
|
||||||
inCredit: num(sideMap.get(c.currency)?.inCredit),
|
inCredit: num(sideMap.get(c.currency)?.inCredit),
|
||||||
})),
|
})),
|
||||||
byDomain: byDomain.map((d) => ({
|
byDomain: byDomain.map((d) => ({
|
||||||
domain: d.domain,
|
domain: d.domain,
|
||||||
currency: d.currency,
|
currency: d.currency,
|
||||||
net: d._sum.amount,
|
net: d.net,
|
||||||
count: d._count._all,
|
count: num(d.count),
|
||||||
})),
|
})),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -589,7 +673,7 @@ export class BillingService {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const years = await this.prisma.$queryRaw<
|
const years = await this.prisma.$queryRaw<
|
||||||
{ year: number; count: bigint | number | string }[]
|
{ year: number; count: RawCount }[]
|
||||||
>`
|
>`
|
||||||
SELECT YEAR(transactionDate) AS year, COUNT(*) AS count
|
SELECT YEAR(transactionDate) AS year, COUNT(*) AS count
|
||||||
FROM transactions WHERE voidedAt IS NULL GROUP BY year ORDER BY year DESC
|
FROM transactions WHERE voidedAt IS NULL GROUP BY year ORDER BY year DESC
|
||||||
@@ -647,9 +731,32 @@ export class BillingService {
|
|||||||
throw new NotFoundException(`Customer ${customerId} not found`);
|
throw new NotFoundException(`Customer ${customerId} not found`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One customer, so the balance floor is a single date rather than the
|
||||||
|
// derived table the aggregate queries join. See NOT_SUPERSEDED: rows before
|
||||||
|
// the opening balance are already inside it, and showing them would both
|
||||||
|
// double the total and make every balanceAfter below wrong.
|
||||||
|
//
|
||||||
|
// This is also what stops FEE ANUAL and fee15 leaking in. They are not in
|
||||||
|
// STATEMENT_EXCLUDED_SOURCE_TABLES — that list exists to reproduce legacy's
|
||||||
|
// DATOS2-only `datosfreak`, and it was letting 2,092 pre-cutover fee rows
|
||||||
|
// across 1,062 customers through, skewing the statement by -5,129,764
|
||||||
|
// against the number those customers have been quoted for years. Dating
|
||||||
|
// rather than source is the right test: a FEE ANUAL row *after* the opening
|
||||||
|
// balance is a real charge and still counts.
|
||||||
|
const floor = await this.prisma.transaction.findFirst({
|
||||||
|
where: {
|
||||||
|
customerId,
|
||||||
|
voidedAt: null,
|
||||||
|
type: { nameEn: BALANCE_FORWARD_TYPE },
|
||||||
|
},
|
||||||
|
orderBy: { transactionDate: "desc" },
|
||||||
|
select: { transactionDate: true },
|
||||||
|
});
|
||||||
|
|
||||||
const rows = await this.prisma.transaction.findMany({
|
const rows = await this.prisma.transaction.findMany({
|
||||||
where: {
|
where: {
|
||||||
customerId,
|
customerId,
|
||||||
|
...(floor ? { transactionDate: { gte: floor.transactionDate } } : {}),
|
||||||
// NULL-safe exclusion. `notIn` alone compiles to SQL `NOT IN`, and
|
// NULL-safe exclusion. `notIn` alone compiles to SQL `NOT IN`, and
|
||||||
// `NULL NOT IN (...)` is NULL, not true — so every app-captured row
|
// `NULL NOT IN (...)` is NULL, not true — so every app-captured row
|
||||||
// (which has no legacySourceTable) silently vanished from the
|
// (which has no legacySourceTable) silently vanished from the
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import { AbilityGuard } from "../auth/ability.guard";
|
|||||||
import { RequireAbility } from "../auth/require-ability.decorator";
|
import { RequireAbility } from "../auth/require-ability.decorator";
|
||||||
import { AuditService } from "../common/audit.service";
|
import { AuditService } from "../common/audit.service";
|
||||||
import { CustomersService } from "./customers.service";
|
import { CustomersService } from "./customers.service";
|
||||||
|
import { NumidService } from "./numid.service";
|
||||||
import { CreateCustomerDto } from "./create-customer.dto";
|
import { CreateCustomerDto } from "./create-customer.dto";
|
||||||
import { UpdateCustomerDto } from "./update-customer.dto";
|
import { UpdateCustomerDto } from "./update-customer.dto";
|
||||||
|
|
||||||
@@ -24,6 +25,7 @@ import { UpdateCustomerDto } from "./update-customer.dto";
|
|||||||
export class CustomersController {
|
export class CustomersController {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly customers: CustomersService,
|
private readonly customers: CustomersService,
|
||||||
|
private readonly numids: NumidService,
|
||||||
private readonly audit: AuditService,
|
private readonly audit: AuditService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
@@ -36,6 +38,13 @@ export class CustomersController {
|
|||||||
return this.customers.stats();
|
return this.customers.stats();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Reusable portal ids, lowest first. Declared above `:id` so the literal
|
||||||
|
* path is not swallowed by the wildcard route. */
|
||||||
|
@Get("numid/candidates")
|
||||||
|
async numidCandidates() {
|
||||||
|
return { candidates: await this.numids.emptyCandidates() };
|
||||||
|
}
|
||||||
|
|
||||||
@Get()
|
@Get()
|
||||||
list(
|
list(
|
||||||
@Query("query") query?: string,
|
@Query("query") query?: string,
|
||||||
@@ -95,4 +104,27 @@ export class CustomersController {
|
|||||||
void this.audit.log(this.actingId(req), "customer.restore", { customerId: id });
|
void this.audit.log(this.actingId(req), "customer.restore", { customerId: id });
|
||||||
return c;
|
return c;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Give this customer a portal NUMid so they can log in to
|
||||||
|
* my.jorgecuadros.com. Idempotent — a customer who already has one gets it
|
||||||
|
* back rather than a second identity.
|
||||||
|
*/
|
||||||
|
@Post(":id/portal-access")
|
||||||
|
@RequireAbility("customer:portal-access")
|
||||||
|
async portalAccess(@Param("id") id: string, @Req() req: Request) {
|
||||||
|
const allocation = await this.numids.allocate(id);
|
||||||
|
if (allocation.origin !== "existing") {
|
||||||
|
// Logged with the origin and the previous holder: a recycled id is the one
|
||||||
|
// case where reading this record later has to answer "whose number was
|
||||||
|
// this before, and was it taken or minted".
|
||||||
|
void this.audit.log(this.actingId(req), "customer.portal-access", {
|
||||||
|
customerId: id,
|
||||||
|
numid: allocation.numid,
|
||||||
|
origin: allocation.origin,
|
||||||
|
previousCustomerId: allocation.previousCustomerId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return allocation;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,13 @@
|
|||||||
import { Module } from "@nestjs/common";
|
import { Module } from "@nestjs/common";
|
||||||
|
import { SettingsModule } from "../settings/settings.module";
|
||||||
import { CustomersController } from "./customers.controller";
|
import { CustomersController } from "./customers.controller";
|
||||||
import { CustomersService } from "./customers.service";
|
import { CustomersService } from "./customers.service";
|
||||||
|
import { NumidService } from "./numid.service";
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
|
imports: [SettingsModule],
|
||||||
controllers: [CustomersController],
|
controllers: [CustomersController],
|
||||||
providers: [CustomersService],
|
providers: [CustomersService, NumidService],
|
||||||
|
exports: [NumidService],
|
||||||
})
|
})
|
||||||
export class CustomersModule {}
|
export class CustomersModule {}
|
||||||
|
|||||||
@@ -0,0 +1,206 @@
|
|||||||
|
import { ConflictException, NotFoundException } from "@nestjs/common";
|
||||||
|
import { Prisma } from "@jorgecuadros/database";
|
||||||
|
import { NumidService } from "./numid.service";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* What matters about the allocator is the two things it must never do: hand the
|
||||||
|
* same id to two customers, and hand out a recycled id while Access can still
|
||||||
|
* take it back. Both are tested here; the emptiness SQL itself is exercised
|
||||||
|
* against real data by scripts/numid-audit.mjs.
|
||||||
|
*/
|
||||||
|
|
||||||
|
interface Options {
|
||||||
|
existingRef?: { legacyId: string } | null;
|
||||||
|
archived?: boolean;
|
||||||
|
missing?: boolean;
|
||||||
|
recycle?: boolean;
|
||||||
|
empty?: { numid: string; refId: string; customerId: string }[];
|
||||||
|
max?: number | null;
|
||||||
|
/** Make the first N create() calls fail the unique key, as a race would. */
|
||||||
|
createConflicts?: number;
|
||||||
|
/** Make updateMany report "nothing matched", as a lost recycle race would. */
|
||||||
|
recycleMisses?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function build(opts: Options = {}) {
|
||||||
|
const created: { legacyId: string }[] = [];
|
||||||
|
let conflictsLeft = opts.createConflicts ?? 0;
|
||||||
|
let missesLeft = opts.recycleMisses ?? 0;
|
||||||
|
|
||||||
|
const prisma = {
|
||||||
|
customer: {
|
||||||
|
findUnique: jest.fn().mockResolvedValue(
|
||||||
|
opts.missing ? null : { id: "cust-new", archivedAt: opts.archived ? new Date() : null },
|
||||||
|
),
|
||||||
|
},
|
||||||
|
customerLegacyRef: {
|
||||||
|
findFirst: jest.fn().mockResolvedValue(opts.existingRef ?? null),
|
||||||
|
updateMany: jest.fn().mockImplementation(() => {
|
||||||
|
if (missesLeft > 0) {
|
||||||
|
missesLeft -= 1;
|
||||||
|
return Promise.resolve({ count: 0 });
|
||||||
|
}
|
||||||
|
return Promise.resolve({ count: 1 });
|
||||||
|
}),
|
||||||
|
create: jest.fn().mockImplementation(({ data }: { data: { legacyId: string } }) => {
|
||||||
|
if (conflictsLeft > 0) {
|
||||||
|
conflictsLeft -= 1;
|
||||||
|
return Promise.reject(
|
||||||
|
new Prisma.PrismaClientKnownRequestError("dup", {
|
||||||
|
code: "P2002",
|
||||||
|
clientVersion: "5",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
created.push(data);
|
||||||
|
return Promise.resolve(data);
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
// Two different raw queries share one mock: the MAX lookup returns a single
|
||||||
|
// {max} row, everything else is the empty-candidate list.
|
||||||
|
$queryRaw: jest.fn().mockImplementation((sql: { strings?: string[]; sql?: string }) => {
|
||||||
|
const text = String((sql as unknown as { sql?: string }).sql ?? "");
|
||||||
|
if (text.includes("MAX(")) return Promise.resolve([{ max: opts.max ?? null }]);
|
||||||
|
return Promise.resolve(opts.empty ?? []);
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
const settings = {
|
||||||
|
numidRecycleEmpty: jest
|
||||||
|
.fn()
|
||||||
|
.mockResolvedValue({ value: opts.recycle ?? false, source: "default" }),
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
service: new NumidService(prisma as never, settings as never),
|
||||||
|
prisma,
|
||||||
|
created,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("NUMid allocation", () => {
|
||||||
|
it("returns the id a customer already holds instead of minting a second one", async () => {
|
||||||
|
// A double-clicked button must not fork the customer's portal identity.
|
||||||
|
const { service, prisma } = build({ existingRef: { legacyId: "501" } });
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).resolves.toEqual({
|
||||||
|
numid: "501",
|
||||||
|
origin: "existing",
|
||||||
|
});
|
||||||
|
expect(prisma.customerLegacyRef.create).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("allocates one past the highest id in the pool", async () => {
|
||||||
|
const { service, created } = build({ max: 1171 });
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).resolves.toEqual({
|
||||||
|
numid: "1172",
|
||||||
|
origin: "new",
|
||||||
|
});
|
||||||
|
expect(created[0]).toMatchObject({
|
||||||
|
sourceSystem: "utilities",
|
||||||
|
sourceTable: "DATGRAL",
|
||||||
|
legacyId: "1172",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("starts at 1 when the pool is empty", async () => {
|
||||||
|
const { service } = build({ max: null });
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).resolves.toMatchObject({ numid: "1" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does NOT recycle while the setting is off, even with candidates free", async () => {
|
||||||
|
// The default has to be the safe one: every reusable id still exists in
|
||||||
|
// Access, and a --sync run reassigns it back to its Access owner.
|
||||||
|
const { service, prisma } = build({
|
||||||
|
max: 1171,
|
||||||
|
empty: [{ numid: "1089", refId: "ref-1089", customerId: "cust-old" }],
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).resolves.toMatchObject({
|
||||||
|
numid: "1172",
|
||||||
|
origin: "new",
|
||||||
|
});
|
||||||
|
expect(prisma.customerLegacyRef.updateMany).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("takes the lowest empty id once recycling is switched on", async () => {
|
||||||
|
const { service, prisma } = build({
|
||||||
|
recycle: true,
|
||||||
|
max: 1171,
|
||||||
|
empty: [
|
||||||
|
{ numid: "1089", refId: "ref-1089", customerId: "cust-old" },
|
||||||
|
{ numid: "1094", refId: "ref-1094", customerId: "cust-other" },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).resolves.toEqual({
|
||||||
|
numid: "1089",
|
||||||
|
origin: "recycled",
|
||||||
|
previousCustomerId: "cust-old",
|
||||||
|
});
|
||||||
|
// Guarded on the owner read a moment ago, so a ref that moved underneath us
|
||||||
|
// matches nothing rather than being stolen.
|
||||||
|
expect(prisma.customerLegacyRef.updateMany).toHaveBeenCalledWith({
|
||||||
|
where: { id: "ref-1089", customerId: "cust-old" },
|
||||||
|
data: { customerId: "cust-new" },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("skips a candidate that someone else took first", async () => {
|
||||||
|
const { service } = build({
|
||||||
|
recycle: true,
|
||||||
|
max: 1171,
|
||||||
|
recycleMisses: 1,
|
||||||
|
empty: [
|
||||||
|
{ numid: "1089", refId: "ref-1089", customerId: "cust-old" },
|
||||||
|
{ numid: "1094", refId: "ref-1094", customerId: "cust-other" },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).resolves.toMatchObject({
|
||||||
|
numid: "1094",
|
||||||
|
origin: "recycled",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to a new id when recycling is on but nothing is free", async () => {
|
||||||
|
const { service } = build({ recycle: true, max: 1171, empty: [] });
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).resolves.toMatchObject({
|
||||||
|
numid: "1172",
|
||||||
|
origin: "new",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries when two writers pick the same id", async () => {
|
||||||
|
// The unique key on (sourceSystem, sourceTable, legacyId) is what decides
|
||||||
|
// the winner; the loser must retry, never overwrite.
|
||||||
|
const { service, prisma } = build({ max: 1171, createConflicts: 1 });
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).resolves.toMatchObject({
|
||||||
|
numid: "1172",
|
||||||
|
origin: "new",
|
||||||
|
});
|
||||||
|
expect(prisma.customerLegacyRef.create).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("gives up loudly rather than looping forever", async () => {
|
||||||
|
const { service } = build({ max: 1171, createConflicts: 99 });
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).rejects.toBeInstanceOf(ConflictException);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses an archived customer", async () => {
|
||||||
|
const { service } = build({ archived: true });
|
||||||
|
|
||||||
|
await expect(service.allocate("cust-new")).rejects.toBeInstanceOf(ConflictException);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("refuses a customer that does not exist", async () => {
|
||||||
|
const { service } = build({ missing: true });
|
||||||
|
|
||||||
|
await expect(service.allocate("nope")).rejects.toBeInstanceOf(NotFoundException);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
import {
|
||||||
|
ConflictException,
|
||||||
|
Injectable,
|
||||||
|
Logger,
|
||||||
|
NotFoundException,
|
||||||
|
} from "@nestjs/common";
|
||||||
|
import { Prisma } from "@jorgecuadros/database";
|
||||||
|
import { PrismaService } from "../prisma/prisma.service";
|
||||||
|
import { SettingsService } from "../settings/settings.service";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Allocation of the portal NUMid — the "Security Number" my.jorgecuadros.com
|
||||||
|
* asks for at login.
|
||||||
|
*
|
||||||
|
* The NUMid is not a column on `Customer`. It is a `CustomerLegacyRef` row with
|
||||||
|
* (sourceSystem='utilities', sourceTable='DATGRAL'), and `CustomersService.create`
|
||||||
|
* deliberately writes none: a natively created customer has no legacy provenance.
|
||||||
|
* The consequence is that every customer created in the staff UI is invisible to
|
||||||
|
* the portal until this service gives them an id.
|
||||||
|
*
|
||||||
|
* WHY THIS IS NOT DONE AT CREATE TIME. Insurance is expected to move to the
|
||||||
|
* platform before utilities, and an insurance-only customer has no reason to hold
|
||||||
|
* a portal identity. Allocating on every create would spend utilities ids — and
|
||||||
|
* the handful of reusable ones — on people who will never log in. So this is an
|
||||||
|
* explicit staff action instead.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** The pair that identifies a portal NUMid. */
|
||||||
|
export const UTILITIES_SYSTEM = "utilities";
|
||||||
|
export const UTILITIES_TABLE = "DATGRAL";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* insurance/DATGRAL is a SEPARATE id space that reuses the same sourceTable name
|
||||||
|
* and runs past 4,000. It must never be read as a NUMid, and never allocated
|
||||||
|
* from: the portal cannot resolve those ids. Every query here filters on BOTH
|
||||||
|
* columns for that reason, never on sourceTable alone. A customer can also hold
|
||||||
|
* more than one insurance ref — 16 of them do, where several insurance rows
|
||||||
|
* folded into one customer — so those are tested with EXISTS rather than joined.
|
||||||
|
*/
|
||||||
|
const POOL = {
|
||||||
|
sourceSystem: UTILITIES_SYSTEM,
|
||||||
|
sourceTable: UTILITIES_TABLE,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export type AllocationOrigin = "existing" | "new" | "recycled";
|
||||||
|
|
||||||
|
export interface Allocation {
|
||||||
|
numid: string;
|
||||||
|
origin: AllocationOrigin;
|
||||||
|
/** Set only on a recycle — the customer the id was taken from. */
|
||||||
|
previousCustomerId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* NUMids that were created and never used, safe for an allocator to take.
|
||||||
|
*
|
||||||
|
* THE TWO OBVIOUS RULES BOTH FIND NOTHING, which is why this one looks the way
|
||||||
|
* it does. "Owns no rows" matches nobody: migration gave all 1,171 NUMids a
|
||||||
|
* property and a transaction. "No transaction in N years" also matches nobody:
|
||||||
|
* every customer carries a synthetic Jan-1 opening-balance row, so everyone
|
||||||
|
* looks active in the current year. That row has to be subtracted before any
|
||||||
|
* activity test means anything, which is what `bf` does below.
|
||||||
|
*
|
||||||
|
* The balance-forward row is matched in two shapes on purpose.
|
||||||
|
* transform_transactions.py:120 mints a type literally named 'BALANCE FORWARD';
|
||||||
|
* databases loaded before that change carry the same rows with typeId NULL,
|
||||||
|
* dated Jan 1, legacySourceTable='datos2'. Matching only the type name floors
|
||||||
|
* nothing on such a database and turns the balance test into a raw lifetime sum
|
||||||
|
* — the double-count that read the whole book as +20.6M MXN in credit before
|
||||||
|
* d173c9e, and which here would mark live customers as empty.
|
||||||
|
*
|
||||||
|
* Services are tested as "any service" rather than "any ACTIVE service": a
|
||||||
|
* deactivated water account is still a record of somebody having lived behind
|
||||||
|
* this id.
|
||||||
|
*
|
||||||
|
* Kept in step with scripts/numid-audit.sql, which reports the same tier for a
|
||||||
|
* human. That script is the reporting copy of this rule; change both together.
|
||||||
|
*/
|
||||||
|
const EMPTY_NUMID_SQL = Prisma.sql`
|
||||||
|
WITH bf AS (
|
||||||
|
SELECT t.id, t.customerId
|
||||||
|
FROM transactions t
|
||||||
|
LEFT JOIN type_transactions tt ON tt.id = t.typeId
|
||||||
|
WHERE t.voidedAt IS NULL
|
||||||
|
AND (
|
||||||
|
tt.nameEn = 'BALANCE FORWARD'
|
||||||
|
OR (t.typeId IS NULL AND MONTH(t.transactionDate) = 1 AND DAY(t.transactionDate) = 1
|
||||||
|
AND t.legacySourceTable = 'datos2')
|
||||||
|
)
|
||||||
|
)
|
||||||
|
SELECT r.legacyId AS numid, r.id AS refId, r.customerId AS customerId
|
||||||
|
FROM customer_legacy_refs r
|
||||||
|
JOIN customers c ON c.id = r.customerId
|
||||||
|
WHERE r.sourceSystem = ${UTILITIES_SYSTEM} AND r.sourceTable = ${UTILITIES_TABLE}
|
||||||
|
AND (c.email IS NULL OR c.email = '')
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM transactions t
|
||||||
|
WHERE t.customerId = c.id AND t.voidedAt IS NULL
|
||||||
|
AND t.id NOT IN (SELECT id FROM bf))
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM transactions t
|
||||||
|
WHERE t.customerId = c.id AND t.voidedAt IS NULL AND t.outstanding = 1)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM property_services ps
|
||||||
|
JOIN properties p ON p.id = ps.propertyId WHERE p.customerId = c.id)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM policies p WHERE p.customerId = c.id)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM vehicles v WHERE v.customerId = c.id)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM trust_accounts ta
|
||||||
|
JOIN properties p ON p.id = ta.propertyId WHERE p.customerId = c.id)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM statement_documents s WHERE s.matchedCustomerId = c.id)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM policy_ocr_documents o WHERE o.matchedCustomerId = c.id)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM email_notification_log e WHERE e.customerId = c.id)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM email_log e WHERE e.customerId = c.id)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM account_status_history a WHERE a.customerId = c.id)
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM customer_legacy_refs i
|
||||||
|
WHERE i.customerId = c.id AND i.sourceSystem = 'insurance')
|
||||||
|
ORDER BY CAST(r.legacyId AS UNSIGNED)`;
|
||||||
|
|
||||||
|
interface EmptyRow {
|
||||||
|
numid: string;
|
||||||
|
refId: string;
|
||||||
|
customerId: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class NumidService {
|
||||||
|
private readonly logger = new Logger(NumidService.name);
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly prisma: PrismaService,
|
||||||
|
private readonly settings: SettingsService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/** The customer's portal id, or null if they have none. */
|
||||||
|
async current(customerId: string): Promise<string | null> {
|
||||||
|
const ref = await this.prisma.customerLegacyRef.findFirst({
|
||||||
|
where: { customerId, ...POOL },
|
||||||
|
select: { legacyId: true },
|
||||||
|
});
|
||||||
|
return ref?.legacyId ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Reusable ids, lowest first. Empty unless recycling is switched on. */
|
||||||
|
async emptyCandidates(): Promise<string[]> {
|
||||||
|
const rows = await this.prisma.$queryRaw<EmptyRow[]>(EMPTY_NUMID_SQL);
|
||||||
|
return rows.map((r) => r.numid);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Give a customer a portal NUMid.
|
||||||
|
*
|
||||||
|
* Idempotent: a customer who already holds one gets it back rather than a
|
||||||
|
* second id, so a double-clicked button cannot fork an identity.
|
||||||
|
*/
|
||||||
|
async allocate(customerId: string): Promise<Allocation> {
|
||||||
|
const customer = await this.prisma.customer.findUnique({
|
||||||
|
where: { id: customerId },
|
||||||
|
select: { id: true, archivedAt: true },
|
||||||
|
});
|
||||||
|
if (!customer) throw new NotFoundException(`Customer ${customerId} not found`);
|
||||||
|
if (customer.archivedAt) {
|
||||||
|
throw new ConflictException(
|
||||||
|
"No se puede asignar un número de portal a un cliente archivado",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const existing = await this.current(customerId);
|
||||||
|
if (existing) return { numid: existing, origin: "existing" };
|
||||||
|
|
||||||
|
const recycle = await this.recycleEnabled();
|
||||||
|
|
||||||
|
// Two writers can pick the same id between the read and the write. The
|
||||||
|
// unique key on (sourceSystem, sourceTable, legacyId) is what actually
|
||||||
|
// decides the winner; the loser retries and takes the next id rather than
|
||||||
|
// silently overwriting. Bounded so a genuinely wedged pool fails loudly.
|
||||||
|
for (let attempt = 0; attempt < 5; attempt++) {
|
||||||
|
try {
|
||||||
|
if (recycle) {
|
||||||
|
const recycled = await this.tryRecycle(customerId);
|
||||||
|
if (recycled) return recycled;
|
||||||
|
}
|
||||||
|
return await this.allocateNext(customerId);
|
||||||
|
} catch (error) {
|
||||||
|
if (!isUniqueViolation(error)) throw error;
|
||||||
|
this.logger.warn(
|
||||||
|
`NUMid allocation for ${customerId} lost a race (attempt ${attempt + 1}), retrying`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new ConflictException(
|
||||||
|
"No se pudo asignar un número de portal; intente de nuevo",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether the recycle tier is live.
|
||||||
|
*
|
||||||
|
* Off by default, and that default is the safe one while Access is still the
|
||||||
|
* utilities master. Every id in the pool ALSO exists in Access DATGRAL, and a
|
||||||
|
* `--sync` migration run upserts refs with ON DUPLICATE KEY UPDATE customerId
|
||||||
|
* (transform_customers.py:327) — so an id recycled today is silently handed
|
||||||
|
* back to its Access owner on the next sync, and the customer who was given it
|
||||||
|
* loses their portal identity. Turn this on once utilities has cut over, or
|
||||||
|
* for ids that have been deleted at the source.
|
||||||
|
*/
|
||||||
|
private async recycleEnabled(): Promise<boolean> {
|
||||||
|
const { value } = await this.settings.numidRecycleEmpty();
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Re-point the lowest empty id at this customer. Null when none is free. */
|
||||||
|
private async tryRecycle(customerId: string): Promise<Allocation | null> {
|
||||||
|
const rows = await this.prisma.$queryRaw<EmptyRow[]>(EMPTY_NUMID_SQL);
|
||||||
|
for (const row of rows) {
|
||||||
|
// Guarded by the owner we just read: if anything moved the ref in the
|
||||||
|
// meantime the update matches nothing and we fall through to the next
|
||||||
|
// candidate rather than stealing an id that is no longer empty.
|
||||||
|
const moved = await this.prisma.customerLegacyRef.updateMany({
|
||||||
|
where: { id: row.refId, customerId: row.customerId },
|
||||||
|
data: { customerId },
|
||||||
|
});
|
||||||
|
if (moved.count === 1) {
|
||||||
|
this.logger.log(
|
||||||
|
`NUMid ${row.numid} recycled from ${row.customerId} to ${customerId}`,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
numid: row.numid,
|
||||||
|
origin: "recycled",
|
||||||
|
previousCustomerId: row.customerId,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One past the highest id in the pool. */
|
||||||
|
private async allocateNext(customerId: string): Promise<Allocation> {
|
||||||
|
const [{ max }] = await this.prisma.$queryRaw<{ max: number | null }[]>(
|
||||||
|
// MAX over a CAST, not over the string: legacyId is VARCHAR, so a plain
|
||||||
|
// MAX returns '999' as the highest of 1,171 rows and the allocator hands
|
||||||
|
// out an id that is already taken.
|
||||||
|
Prisma.sql`SELECT MAX(CAST(legacyId AS UNSIGNED)) AS max
|
||||||
|
FROM customer_legacy_refs
|
||||||
|
WHERE sourceSystem = ${UTILITIES_SYSTEM} AND sourceTable = ${UTILITIES_TABLE}`,
|
||||||
|
);
|
||||||
|
const numid = String(Number(max ?? 0) + 1);
|
||||||
|
await this.prisma.customerLegacyRef.create({
|
||||||
|
data: { customerId, ...POOL, legacyId: numid },
|
||||||
|
});
|
||||||
|
return { numid, origin: "new" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isUniqueViolation(error: unknown): boolean {
|
||||||
|
return (
|
||||||
|
error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002"
|
||||||
|
);
|
||||||
|
}
|
||||||
+13
-1
@@ -60,7 +60,19 @@ async function bootstrap() {
|
|||||||
app.use(passport.initialize());
|
app.use(passport.initialize());
|
||||||
app.use(passport.session());
|
app.use(passport.session());
|
||||||
|
|
||||||
app.enableCors({ credentials: true, origin: process.env.WEB_ORIGIN ?? "http://localhost:3000" });
|
// The same deployment is reached under several origins — the office LAN IP,
|
||||||
|
// the tailnet name, the demo domain — and the browser derives the API origin
|
||||||
|
// from whichever one served the page (apps/web/src/lib/api.ts). So WEB_ORIGIN
|
||||||
|
// is a comma-separated LIST, not a single value. A request whose Origin is
|
||||||
|
// not listed gets no CORS headers and the credentialed fetch fails, so add an
|
||||||
|
// entry when a new way of reaching the app is introduced. Same-origin setups
|
||||||
|
// (web and API behind one proxy) never hit CORS at all.
|
||||||
|
const webOrigins = (process.env.WEB_ORIGIN ?? "http://localhost:3000")
|
||||||
|
.split(",")
|
||||||
|
.map((o) => o.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
app.enableCors({ credentials: true, origin: webOrigins });
|
||||||
|
|
||||||
const port = process.env.PORT ? Number(process.env.PORT) : 3001;
|
const port = process.env.PORT ? Number(process.env.PORT) : 3001;
|
||||||
await app.listen(port);
|
await app.listen(port);
|
||||||
|
|||||||
@@ -104,6 +104,24 @@ export class OpsController {
|
|||||||
return this.replication.status();
|
return this.replication.status();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Full row-by-row comparison of the customer-visible tables against the master.
|
||||||
|
*
|
||||||
|
* POST rather than GET despite reading nothing: it is a full scan of both
|
||||||
|
* servers and must not be something a browser prefetch, a retry, or a refresh
|
||||||
|
* can set off. Audited for the same reason — it is a deliberate, costly act,
|
||||||
|
* and "who ran this while the site was slow" is a question worth answering.
|
||||||
|
*/
|
||||||
|
@Post("replication/verify")
|
||||||
|
async verifyReplication(@Req() req: Request) {
|
||||||
|
const result = await this.replication.verify();
|
||||||
|
void this.audit.log(this.actingId(req), "ops.replication.verify", {
|
||||||
|
identical: result.identical,
|
||||||
|
elapsedMs: result.elapsedMs,
|
||||||
|
});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
@Get("jobs")
|
@Get("jobs")
|
||||||
listJobs() {
|
listJobs() {
|
||||||
return this.ops.listJobs();
|
return this.ops.listJobs();
|
||||||
@@ -117,11 +135,17 @@ export class OpsController {
|
|||||||
@Post("jobs")
|
@Post("jobs")
|
||||||
async startJob(@Body() dto: StartJobDto, @Req() req: Request) {
|
async startJob(@Body() dto: StartJobDto, @Req() req: Request) {
|
||||||
const userId = this.actingId(req);
|
const userId = this.actingId(req);
|
||||||
const job = await this.ops.startJob(dto.kind, { file: dto.file }, userId);
|
const job = await this.ops.startJob(
|
||||||
|
dto.kind,
|
||||||
|
{ file: dto.file, forceFull: dto.forceFull },
|
||||||
|
userId,
|
||||||
|
);
|
||||||
void this.audit.log(userId, "ops.job.start", {
|
void this.audit.log(userId, "ops.job.start", {
|
||||||
jobId: job.id,
|
jobId: job.id,
|
||||||
kind: dto.kind,
|
kind: dto.kind,
|
||||||
file: dto.file,
|
file: dto.file,
|
||||||
|
// Recorded because this is the flag that authorised deleting native rows.
|
||||||
|
forceFull: dto.forceFull,
|
||||||
});
|
});
|
||||||
return job;
|
return job;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -415,12 +415,19 @@ export class OpsService implements OnModuleInit {
|
|||||||
const out = shq(path.join(this.backupDir, file));
|
const out = shq(path.join(this.backupDir, file));
|
||||||
const py = await this.pythonBin();
|
const py = await this.pythonBin();
|
||||||
const runAll = shq(path.join(this.migrationDir, "run_all.py"));
|
const runAll = shq(path.join(this.migrationDir, "run_all.py"));
|
||||||
|
// run_all.py runs native_guard.py before it truncates anything and exits
|
||||||
|
// without touching the database when the target holds rows that only
|
||||||
|
// exist here — allocated portal NUMids, app-created customers, OCR
|
||||||
|
// captures. --force-full is what the operator ticks to delete them
|
||||||
|
// anyway; without it the job fails with the list.
|
||||||
|
const force = params.forceFull === true;
|
||||||
const cmd =
|
const cmd =
|
||||||
`${PIPEFAIL}echo '== Respaldo de seguridad previo ==' && ` +
|
`${PIPEFAIL}echo '== Respaldo de seguridad previo ==' && ` +
|
||||||
`${this.dumpCommand(flags, db, out)} && ` +
|
`${this.dumpCommand(flags, db, out)} && ` +
|
||||||
`echo '== Reimportación desde carpeta de ingesta ==' && ` +
|
`echo '== Reimportación desde carpeta de ingesta ==' && ` +
|
||||||
`${shq(py)} ${runAll} --env ${shq(this.migrationEnv)} --stage`;
|
`${shq(py)} ${runAll} --env ${shq(this.migrationEnv)} --stage` +
|
||||||
return { cmd, resolvedParams: { safetyBackup: file } };
|
(force ? " --force-full" : "");
|
||||||
|
return { cmd, resolvedParams: { safetyBackup: file, forceFull: force } };
|
||||||
}
|
}
|
||||||
|
|
||||||
throw new BadRequestException(`Operación no soportada: ${kind}`);
|
throw new BadRequestException(`Operación no soportada: ${kind}`);
|
||||||
|
|||||||
@@ -43,6 +43,58 @@ export interface ApplyProgress {
|
|||||||
percent: number | null;
|
percent: number | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How far the replica's executed history is from the master's, in transactions.
|
||||||
|
*
|
||||||
|
* This is the check `SHOW REPLICA STATUS` cannot give you, and it is stronger
|
||||||
|
* than everything else on the card for one specific reason: every other field is
|
||||||
|
* self-reported by the replica. `Seconds_Behind_Source` reads 0 both when there
|
||||||
|
* is genuinely nothing to apply AND when the I/O thread is disconnected — with
|
||||||
|
* no incoming event there is nothing to measure staleness against, so a dead
|
||||||
|
* link reports as perfectly current. `GTID_SUBTRACT(master, replica)` asks the
|
||||||
|
* master what it has done and the replica what it has applied, so a silent
|
||||||
|
* disconnect shows up immediately as a growing number.
|
||||||
|
*/
|
||||||
|
export interface GtidDrift {
|
||||||
|
/** Transactions the master executed that the replica has not. 0 = identical. */
|
||||||
|
missingTransactions: number;
|
||||||
|
/** The missing GTID set verbatim. Null when nothing is missing. */
|
||||||
|
missingGtidSet: string | null;
|
||||||
|
/**
|
||||||
|
* Transactions in the replica's `gtid_executed` under its OWN server UUID —
|
||||||
|
* writes that happened here and exist nowhere on the master.
|
||||||
|
*
|
||||||
|
* Reported, never alarmed on. A non-zero count is the expected residue of the
|
||||||
|
* seed load: restoring a dump executes its statements locally, and they take
|
||||||
|
* GTIDs from this server's UUID. They never propagate (`log_replica_updates`
|
||||||
|
* is off and nothing sources from this node), so they are harmless — right up
|
||||||
|
* until someone tries to promote this box, where they become a real divergence.
|
||||||
|
*/
|
||||||
|
localTransactions: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One table's row count and content fingerprint, on one side of the link. */
|
||||||
|
export interface TableFingerprint {
|
||||||
|
table: string;
|
||||||
|
masterRows: number;
|
||||||
|
replicaRows: number;
|
||||||
|
/** Order-independent checksum over every column of every row. */
|
||||||
|
masterChecksum: string;
|
||||||
|
replicaChecksum: string;
|
||||||
|
matches: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VerifyResult {
|
||||||
|
/** True only when every table matched on both count and checksum. */
|
||||||
|
identical: boolean;
|
||||||
|
tables: TableFingerprint[];
|
||||||
|
/** Set instead of `tables` when the comparison could not be run at all. */
|
||||||
|
problem: string | null;
|
||||||
|
checkedAt: string;
|
||||||
|
/** Wall-clock cost, because this is a full scan and the caller should see it. */
|
||||||
|
elapsedMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface ReplicationStatus {
|
export interface ReplicationStatus {
|
||||||
/** false when the replica is not configured for this environment at all. */
|
/** false when the replica is not configured for this environment at all. */
|
||||||
configured: boolean;
|
configured: boolean;
|
||||||
@@ -58,11 +110,31 @@ export interface ReplicationStatus {
|
|||||||
sourceHost: string | null;
|
sourceHost: string | null;
|
||||||
/** Relay-log apply progress. Null when the status output has no positions. */
|
/** Relay-log apply progress. Null when the status output has no positions. */
|
||||||
apply: ApplyProgress | null;
|
apply: ApplyProgress | null;
|
||||||
|
/** GTID comparison against the master. Null when the master was unreachable. */
|
||||||
|
drift: GtidDrift | null;
|
||||||
/** Human-readable reason when healthy is false. */
|
/** Human-readable reason when healthy is false. */
|
||||||
problem: string | null;
|
problem: string | null;
|
||||||
checkedAt: string;
|
checkedAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The tables `my.jorgecuadros.com` reads through the `web_reader` grant.
|
||||||
|
*
|
||||||
|
* This list is the verification surface, not the replication surface — the
|
||||||
|
* replica carries the whole schema. These are the eight whose divergence would
|
||||||
|
* actually be visible to a customer, so they are the ones worth a full scan.
|
||||||
|
*/
|
||||||
|
export const REPLICATED_TABLES = [
|
||||||
|
"transactions",
|
||||||
|
"customers",
|
||||||
|
"customer_legacy_refs",
|
||||||
|
"type_transactions",
|
||||||
|
"exchange_rates",
|
||||||
|
"properties",
|
||||||
|
"property_services",
|
||||||
|
"trust_accounts",
|
||||||
|
] as const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reports whether the my.jorgecuadros.com read replica is still replicating.
|
* Reports whether the my.jorgecuadros.com read replica is still replicating.
|
||||||
*
|
*
|
||||||
@@ -99,6 +171,7 @@ export class ReplicationService {
|
|||||||
lastSqlError: null,
|
lastSqlError: null,
|
||||||
sourceHost: null,
|
sourceHost: null,
|
||||||
apply: null,
|
apply: null,
|
||||||
|
drift: null,
|
||||||
problem: null,
|
problem: null,
|
||||||
checkedAt: now,
|
checkedAt: now,
|
||||||
};
|
};
|
||||||
@@ -109,33 +182,7 @@ export class ReplicationService {
|
|||||||
|
|
||||||
let raw: string;
|
let raw: string;
|
||||||
try {
|
try {
|
||||||
// --ssl is required: the replica sets require_secure_transport=ON.
|
raw = await this.onReplica("SHOW REPLICA STATUS\\G");
|
||||||
//
|
|
||||||
// --ssl-verify-server-cert=0 is deliberate and is NOT the same trade-off
|
|
||||||
// the website makes. This hop never leaves Tailscale — the replica is
|
|
||||||
// reached on its CGNAT tailnet address and its firewall admits only this
|
|
||||||
// host — so WireGuard already authenticates the peer. The DreamHost leg
|
|
||||||
// crosses the public internet and therefore pins the CA instead. The
|
|
||||||
// client here is MariaDB's, which rejects our self-signed CA outright
|
|
||||||
// unless it is handed the CA file, which would mean shipping a cert into
|
|
||||||
// this image for a link that is already authenticated.
|
|
||||||
const { stdout } = await exec(
|
|
||||||
"mysql",
|
|
||||||
[
|
|
||||||
`--host=${host}`,
|
|
||||||
`--user=${user}`,
|
|
||||||
"--ssl",
|
|
||||||
"--ssl-verify-server-cert=0",
|
|
||||||
"--connect-timeout=5",
|
|
||||||
"-e",
|
|
||||||
"SHOW REPLICA STATUS\\G",
|
|
||||||
],
|
|
||||||
{
|
|
||||||
env: { ...process.env, MYSQL_PWD: password },
|
|
||||||
timeout: 15_000,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
raw = stdout;
|
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
const msg = e instanceof Error ? e.message : String(e);
|
const msg = e instanceof Error ? e.message : String(e);
|
||||||
this.logger.warn(`no se pudo consultar la réplica: ${msg}`);
|
this.logger.warn(`no se pudo consultar la réplica: ${msg}`);
|
||||||
@@ -189,10 +236,341 @@ export class ReplicationService {
|
|||||||
// alarming on it would cry wolf. It is here to answer "is it moving?"
|
// alarming on it would cry wolf. It is here to answer "is it moving?"
|
||||||
// when the lag counter is stuck.
|
// when the lag counter is stuck.
|
||||||
apply: applyProgress(raw),
|
apply: applyProgress(raw),
|
||||||
|
// Also reported rather than alarmed on, for the same reason: a busy master
|
||||||
|
// is always a few transactions ahead for the instant they are in flight.
|
||||||
|
// Null rather than zero when the master could not be reached — "unknown"
|
||||||
|
// and "identical" must not render the same.
|
||||||
|
drift: await this.gtidDrift(),
|
||||||
problem,
|
problem,
|
||||||
checkedAt: now,
|
checkedAt: now,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compare executed history between master and replica.
|
||||||
|
*
|
||||||
|
* Two round trips: ask the master what it has executed, then ask the replica
|
||||||
|
* to subtract its own history from that. The subtraction runs on the replica
|
||||||
|
* rather than in TypeScript because `GTID_SUBTRACT` already implements the
|
||||||
|
* interval algebra correctly, and reimplementing set subtraction over binlog
|
||||||
|
* ranges is exactly the kind of thing that looks right and is wrong at the
|
||||||
|
* boundaries.
|
||||||
|
*
|
||||||
|
* @returns null on any failure — a broken drift check must never be mistaken
|
||||||
|
* for a healthy zero.
|
||||||
|
*/
|
||||||
|
private async gtidDrift(): Promise<GtidDrift | null> {
|
||||||
|
try {
|
||||||
|
const masterGtid = (await this.onMaster("SELECT @@gtid_executed")).trim();
|
||||||
|
|
||||||
|
// GTID sets are UUIDs, digits, colons, commas, hyphens, whitespace and
|
||||||
|
// (since 8.4) alphanumeric tags. Nothing else is legal, so rejecting
|
||||||
|
// anything outside that alphabet is a whitelist, not a blacklist: with no
|
||||||
|
// quote and no backslash able to survive it, the value cannot escape the
|
||||||
|
// string literal it is interpolated into below.
|
||||||
|
if (masterGtid && !/^[0-9a-fA-F:,\s_-]+$/.test(masterGtid)) {
|
||||||
|
this.logger.warn("gtid_executed del maestro con formato inesperado");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// An empty set means the master has GTID mode off, and there is nothing
|
||||||
|
// meaningful to compare.
|
||||||
|
if (!masterGtid) return null;
|
||||||
|
|
||||||
|
const flat = masterGtid.replace(/\s+/g, "");
|
||||||
|
// Every GTID set is flattened with REPLACE before it leaves the server.
|
||||||
|
// MySQL wraps `gtid_executed` across lines once it holds more than one
|
||||||
|
// source UUID, and this is read back as tab-separated columns — an
|
||||||
|
// embedded newline would split one row into two and silently truncate the
|
||||||
|
// set at the first UUID.
|
||||||
|
const out = await this.onReplica(
|
||||||
|
"SELECT REPLACE(GTID_SUBTRACT(" +
|
||||||
|
`'${flat}', @@gtid_executed), '\\n', ''), ` +
|
||||||
|
"@@server_uuid, REPLACE(@@gtid_executed, '\\n', '')",
|
||||||
|
["-N"],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Trailing newline only — never `.trim()`. When nothing is missing the
|
||||||
|
// first column is the empty string, so the line begins with a tab, and
|
||||||
|
// trimming it would shift every column one position left and report the
|
||||||
|
// replica's own UUID as the missing GTID set.
|
||||||
|
const [missingSet = "", serverUuid = "", executed = ""] = out
|
||||||
|
.replace(/\r?\n+$/, "")
|
||||||
|
.split("\t");
|
||||||
|
|
||||||
|
return {
|
||||||
|
missingTransactions: countGtids(missingSet),
|
||||||
|
missingGtidSet: missingSet || null,
|
||||||
|
localTransactions: countGtids(gtidsForUuid(executed, serverUuid)),
|
||||||
|
};
|
||||||
|
} catch (e) {
|
||||||
|
const msg = e instanceof Error ? e.message : String(e);
|
||||||
|
this.logger.warn(`no se pudo comparar GTIDs con el maestro: ${msg}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Full-scan comparison of the customer-visible tables on both sides.
|
||||||
|
*
|
||||||
|
* Deliberately NOT part of `status()`: this reads every row of every table in
|
||||||
|
* `REPLICATED_TABLES` on both servers, so it belongs behind a button, not a
|
||||||
|
* 30-second poll.
|
||||||
|
*
|
||||||
|
* It answers the one question GTID drift cannot. GTIDs prove the replica
|
||||||
|
* applied every transaction the master produced; they say nothing about rows
|
||||||
|
* changed on the replica by some other route. A local write is invisible to
|
||||||
|
* every other field on the card and shows up here as a checksum mismatch.
|
||||||
|
*/
|
||||||
|
async verify(): Promise<VerifyResult> {
|
||||||
|
const started = Date.now();
|
||||||
|
const base: VerifyResult = {
|
||||||
|
identical: false,
|
||||||
|
tables: [],
|
||||||
|
problem: null,
|
||||||
|
checkedAt: new Date().toISOString(),
|
||||||
|
elapsedMs: 0,
|
||||||
|
};
|
||||||
|
|
||||||
|
let sql: string;
|
||||||
|
try {
|
||||||
|
sql = await this.fingerprintSql();
|
||||||
|
} catch (e) {
|
||||||
|
const msg = e instanceof Error ? e.message : String(e);
|
||||||
|
return { ...base, problem: `No se pudo leer el esquema: ${msg}`, elapsedMs: Date.now() - started };
|
||||||
|
}
|
||||||
|
|
||||||
|
let masterOut: string;
|
||||||
|
let replicaOut: string;
|
||||||
|
try {
|
||||||
|
// Sequential, not parallel. Running both at once would have the master
|
||||||
|
// scan under the replica's own read load only sometimes, which makes a
|
||||||
|
// slow run hard to attribute; and the boxes are small enough that two
|
||||||
|
// concurrent full scans is a real memory event on the 946MB replica.
|
||||||
|
masterOut = await this.onMaster(sql);
|
||||||
|
replicaOut = await this.onReplica(sql);
|
||||||
|
} catch (e) {
|
||||||
|
const msg = e instanceof Error ? e.message : String(e);
|
||||||
|
return { ...base, problem: `No se pudo comparar: ${msg}`, elapsedMs: Date.now() - started };
|
||||||
|
}
|
||||||
|
|
||||||
|
const master = parseFingerprints(masterOut);
|
||||||
|
const replica = parseFingerprints(replicaOut);
|
||||||
|
|
||||||
|
const tables: TableFingerprint[] = REPLICATED_TABLES.map((table) => {
|
||||||
|
const m = master.get(table);
|
||||||
|
const r = replica.get(table);
|
||||||
|
return {
|
||||||
|
table,
|
||||||
|
masterRows: m?.rows ?? -1,
|
||||||
|
replicaRows: r?.rows ?? -1,
|
||||||
|
masterChecksum: m?.checksum ?? "?",
|
||||||
|
replicaChecksum: r?.checksum ?? "?",
|
||||||
|
// Both sides must have answered. A missing row on either side is a
|
||||||
|
// mismatch, never a pass — `undefined === undefined` would otherwise
|
||||||
|
// report two failed reads as agreement.
|
||||||
|
matches:
|
||||||
|
m !== undefined && r !== undefined && m.rows === r.rows && m.checksum === r.checksum,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
identical: tables.every((t) => t.matches),
|
||||||
|
tables,
|
||||||
|
problem: null,
|
||||||
|
checkedAt: base.checkedAt,
|
||||||
|
elapsedMs: Date.now() - started,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the count+checksum query from the live column list.
|
||||||
|
*
|
||||||
|
* The columns come from `information_schema` on the master rather than being
|
||||||
|
* hardcoded, so the check keeps covering the whole row after a migration adds
|
||||||
|
* one. Reading the schema from the master is safe by construction: if the two
|
||||||
|
* schemas had diverged, replication would already be broken.
|
||||||
|
*/
|
||||||
|
private async fingerprintSql(): Promise<string> {
|
||||||
|
const list = REPLICATED_TABLES.map((t) => `'${t}'`).join(",");
|
||||||
|
const raw = await this.onMaster(
|
||||||
|
"SELECT CONCAT(TABLE_NAME, '\\t', COLUMN_NAME) FROM information_schema.COLUMNS " +
|
||||||
|
`WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (${list}) ` +
|
||||||
|
"ORDER BY TABLE_NAME, ORDINAL_POSITION",
|
||||||
|
["-N"],
|
||||||
|
);
|
||||||
|
|
||||||
|
const cols = new Map<string, string[]>();
|
||||||
|
for (const line of raw.split("\n")) {
|
||||||
|
const [table, column] = line.trim().split("\t");
|
||||||
|
if (!table || !column) continue;
|
||||||
|
cols.set(table, [...(cols.get(table) ?? []), column]);
|
||||||
|
}
|
||||||
|
|
||||||
|
const selects = REPLICATED_TABLES.map((table) => {
|
||||||
|
const columns = cols.get(table);
|
||||||
|
if (!columns?.length) throw new Error(`tabla ${table} sin columnas`);
|
||||||
|
// CONVERT(... USING binary), never CAST(... AS CHAR).
|
||||||
|
//
|
||||||
|
// CAST to CHAR transcodes into the *connection* character set, which is
|
||||||
|
// not the same on the two servers: the mysql client inside the master's
|
||||||
|
// container negotiates latin1, while the replica's negotiates utf8mb4.
|
||||||
|
// Every accented character in a Mexican name, street or note therefore
|
||||||
|
// hashes to different bytes on each side, and the comparison reports a
|
||||||
|
// permanent mismatch on exactly the tables that hold free text — a
|
||||||
|
// verification tool that always cries wolf, which is worse than none.
|
||||||
|
// Comparing the stored bytes sidesteps the session entirely. (Verified
|
||||||
|
// 2026-08-06: with CAST, `customers.name` gave 3344437324815 vs
|
||||||
|
// 3339150372121; with CONVERT both give 3339150372121.)
|
||||||
|
//
|
||||||
|
// 0x1f (unit separator) joins the columns and 0x1e (record separator)
|
||||||
|
// stands in for NULL. Both matter: CONCAT_WS *skips* NULLs rather than
|
||||||
|
// emitting an empty field, so without a placeholder the rows
|
||||||
|
// ('a', NULL, 'b') and ('a', 'b', NULL) produce the same string and a
|
||||||
|
// column-shifting bug would checksum as identical.
|
||||||
|
const expr = columns
|
||||||
|
.map((c) => `IFNULL(CONVERT(\`${c}\` USING binary), 0x1e)`)
|
||||||
|
.join(", 0x1f, ");
|
||||||
|
// SUM, not a running hash: addition is commutative, so the result does not
|
||||||
|
// depend on the order rows come back in. The two servers have no reason to
|
||||||
|
// scan in the same order and are not asked to.
|
||||||
|
return (
|
||||||
|
`SELECT '${table}' AS t, COUNT(*) AS n, ` +
|
||||||
|
`IFNULL(SUM(CRC32(CONCAT_WS(0x1f, ${expr}))), 0) AS c FROM \`${table}\``
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
return selects.join(" UNION ALL ");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ------------------------------------------------------------ plumbing */
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a statement on the replica.
|
||||||
|
*
|
||||||
|
* --ssl is required: the replica sets require_secure_transport=ON.
|
||||||
|
*
|
||||||
|
* --ssl-verify-server-cert=0 is deliberate and is NOT the same trade-off the
|
||||||
|
* website makes. This hop never leaves Tailscale — the replica is reached on
|
||||||
|
* its CGNAT tailnet address and the tailnet ACL admits only this host — so
|
||||||
|
* WireGuard already authenticates the peer. The DreamHost leg crosses the
|
||||||
|
* public internet and therefore pins the CA instead. The client here is
|
||||||
|
* MariaDB's, which rejects our self-signed CA outright unless it is handed the
|
||||||
|
* CA file, which would mean shipping a cert into this image for a link that is
|
||||||
|
* already authenticated.
|
||||||
|
*/
|
||||||
|
private async onReplica(sql: string, extra: string[] = []): Promise<string> {
|
||||||
|
const host = process.env.REPLICA_DB_HOST!;
|
||||||
|
const user = process.env.REPLICA_DB_USER!;
|
||||||
|
const password = process.env.REPLICA_DB_PASS!;
|
||||||
|
const { stdout } = await exec(
|
||||||
|
"mysql",
|
||||||
|
[
|
||||||
|
`--host=${host}`,
|
||||||
|
`--user=${user}`,
|
||||||
|
"--ssl",
|
||||||
|
"--ssl-verify-server-cert=0",
|
||||||
|
"--connect-timeout=5",
|
||||||
|
...extra,
|
||||||
|
"-e",
|
||||||
|
sql,
|
||||||
|
],
|
||||||
|
{ env: { ...process.env, MYSQL_PWD: password }, timeout: VERIFY_TIMEOUT_MS },
|
||||||
|
);
|
||||||
|
return stdout;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run a statement on the master, using the application's own DATABASE_URL.
|
||||||
|
*
|
||||||
|
* The app credential is enough here on purpose — everything this class sends
|
||||||
|
* to the master is a SELECT against `information_schema` or a system variable.
|
||||||
|
* Reaching for OPS_DB_ADMIN_* the way OpsService does would hand a monitoring
|
||||||
|
* read path a credential that can also restore a dump.
|
||||||
|
*/
|
||||||
|
private async onMaster(sql: string, extra: string[] = []): Promise<string> {
|
||||||
|
const raw = process.env.DATABASE_URL;
|
||||||
|
if (!raw) throw new Error("DATABASE_URL no está configurada");
|
||||||
|
const u = new URL(raw);
|
||||||
|
const { stdout } = await exec(
|
||||||
|
"mysql",
|
||||||
|
[
|
||||||
|
`--host=${u.hostname}`,
|
||||||
|
`--port=${u.port || "3306"}`,
|
||||||
|
`--user=${decodeURIComponent(u.username)}`,
|
||||||
|
"--connect-timeout=5",
|
||||||
|
...extra,
|
||||||
|
"-N",
|
||||||
|
"-e",
|
||||||
|
sql,
|
||||||
|
u.pathname.replace(/^\//, ""),
|
||||||
|
],
|
||||||
|
{
|
||||||
|
env: { ...process.env, MYSQL_PWD: decodeURIComponent(u.password) },
|
||||||
|
timeout: VERIFY_TIMEOUT_MS,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
return stdout;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Full scans on a 1-vCPU replica are not fast; 15s would cut them off. */
|
||||||
|
const VERIFY_TIMEOUT_MS = 120_000;
|
||||||
|
|
||||||
|
/** Parse the `t\tn\tc` rows the fingerprint query emits under `mysql -N`. */
|
||||||
|
function parseFingerprints(raw: string): Map<string, { rows: number; checksum: string }> {
|
||||||
|
const out = new Map<string, { rows: number; checksum: string }>();
|
||||||
|
for (const line of raw.split("\n")) {
|
||||||
|
const [table, n, c] = line.trim().split("\t");
|
||||||
|
if (!table || n === undefined || c === undefined) continue;
|
||||||
|
const rows = Number(n);
|
||||||
|
if (!Number.isFinite(rows)) continue;
|
||||||
|
// The checksum stays a string. Sums of CRC32 over 40k rows exceed 2^53, so
|
||||||
|
// parsing it as a number would round and make distinct tables compare equal.
|
||||||
|
out.set(table, { rows, checksum: c });
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Count the transactions in a GTID set.
|
||||||
|
*
|
||||||
|
* Exported for testing. The format is `uuid[:tag]:interval[:interval]...`,
|
||||||
|
* comma-separated, where an interval is `N` or `N-M` inclusive at both ends —
|
||||||
|
* so `1-5` is five transactions, not four.
|
||||||
|
*
|
||||||
|
* MySQL 8.4 added an optional alphanumeric tag between the UUID and the first
|
||||||
|
* interval. It is skipped rather than parsed: any segment that is not a number
|
||||||
|
* or a number range is not an interval, whatever else it may be.
|
||||||
|
*/
|
||||||
|
export function countGtids(set: string): number {
|
||||||
|
if (!set.trim()) return 0;
|
||||||
|
let total = 0;
|
||||||
|
for (const group of set.split(",")) {
|
||||||
|
for (const part of group.trim().split(":").slice(1)) {
|
||||||
|
const m = /^(\d+)(?:-(\d+))?$/.exec(part.trim());
|
||||||
|
if (!m) continue;
|
||||||
|
const from = Number(m[1]);
|
||||||
|
const to = m[2] === undefined ? from : Number(m[2]);
|
||||||
|
if (Number.isFinite(from) && Number.isFinite(to) && to >= from) total += to - from + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Narrow a GTID set to the intervals belonging to one server UUID.
|
||||||
|
*
|
||||||
|
* Exported for testing. Used to isolate the replica's own writes from the
|
||||||
|
* history it replicated, which are interleaved in the same `gtid_executed`.
|
||||||
|
*/
|
||||||
|
export function gtidsForUuid(set: string, uuid: string): string {
|
||||||
|
if (!uuid.trim()) return "";
|
||||||
|
const wanted = uuid.trim().toLowerCase();
|
||||||
|
return set
|
||||||
|
.split(",")
|
||||||
|
.map((g) => g.trim())
|
||||||
|
.filter((g) => g.toLowerCase().startsWith(`${wanted}:`))
|
||||||
|
.join(",");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,4 +1,9 @@
|
|||||||
import { applyProgress, replicaField } from "./replication.service";
|
import {
|
||||||
|
applyProgress,
|
||||||
|
countGtids,
|
||||||
|
gtidsForUuid,
|
||||||
|
replicaField,
|
||||||
|
} from "./replication.service";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verbatim shape of `SHOW REPLICA STATUS\G` from the live replica, trimmed to
|
* Verbatim shape of `SHOW REPLICA STATUS\G` from the live replica, trimmed to
|
||||||
@@ -178,3 +183,72 @@ describe("applyProgress", () => {
|
|||||||
expect(applyProgress(positions("binlog.000042", "NULL", "binlog.000042", 400))).toBeNull();
|
expect(applyProgress(positions("binlog.000042", "NULL", "binlog.000042", 400))).toBeNull();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Real GTID sets from the live pair, captured 2026-08-06. The replica's own
|
||||||
|
* server UUID (3b103283…) carries the transactions the seed dump load executed
|
||||||
|
* locally; the master's UUID (defc34e2…) carries the replicated history.
|
||||||
|
*/
|
||||||
|
const REPLICA_EXECUTED =
|
||||||
|
"3b103283-8f15-11f1-a52b-020017027b33:1-513," +
|
||||||
|
"defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-525";
|
||||||
|
const REPLICA_UUID = "3b103283-8f15-11f1-a52b-020017027b33";
|
||||||
|
|
||||||
|
describe("countGtids", () => {
|
||||||
|
it("counts an inclusive range at both ends", () => {
|
||||||
|
// 1-5 is five transactions. Off-by-one here understates the gap, which is
|
||||||
|
// the direction that hides a problem.
|
||||||
|
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-5")).toBe(5);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("counts a bare single transaction", () => {
|
||||||
|
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:7")).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("sums several intervals under one UUID", () => {
|
||||||
|
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:1-5:8:10-12")).toBe(9);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("sums across UUIDs, including the wrapped form MySQL prints", () => {
|
||||||
|
expect(countGtids(REPLICA_EXECUTED)).toBe(513 + 525);
|
||||||
|
// `gtid_executed` comes back wrapped once it holds more than one UUID.
|
||||||
|
expect(countGtids(REPLICA_EXECUTED.replace(",", ",\n"))).toBe(513 + 525);
|
||||||
|
});
|
||||||
|
|
||||||
|
/** An empty subtraction result is the caught-up case and must be zero. */
|
||||||
|
it("returns 0 for an empty or blank set", () => {
|
||||||
|
expect(countGtids("")).toBe(0);
|
||||||
|
expect(countGtids(" \n ")).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* MySQL 8.4 allows an alphanumeric tag between the UUID and the intervals.
|
||||||
|
* It is not an interval and must not be counted as one.
|
||||||
|
*/
|
||||||
|
it("skips a tag without counting it", () => {
|
||||||
|
expect(countGtids("defc34e2-8c5d-11f1-8e58-52c4c853bce8:mytag:1-3")).toBe(3);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("gtidsForUuid", () => {
|
||||||
|
it("isolates the replica's own transactions from the replicated history", () => {
|
||||||
|
expect(countGtids(gtidsForUuid(REPLICA_EXECUTED, REPLICA_UUID))).toBe(513);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns nothing for a UUID that is not in the set", () => {
|
||||||
|
expect(gtidsForUuid(REPLICA_EXECUTED, "00000000-0000-0000-0000-000000000000")).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The colon matters. Without it a UUID prefix would match a longer UUID that
|
||||||
|
* merely starts the same way, and the replica's local writes would be
|
||||||
|
* over-reported.
|
||||||
|
*/
|
||||||
|
it("does not match on a bare prefix", () => {
|
||||||
|
expect(gtidsForUuid(REPLICA_EXECUTED, "3b103283")).toBe("");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns nothing when the UUID is blank", () => {
|
||||||
|
expect(gtidsForUuid(REPLICA_EXECUTED, "")).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { IsEnum, IsOptional, IsString } from "class-validator";
|
import { IsBoolean, IsEnum, IsOptional, IsString } from "class-validator";
|
||||||
import { OpsJobKind } from "@jorgecuadros/database";
|
import { OpsJobKind } from "@jorgecuadros/database";
|
||||||
|
|
||||||
export class StartJobDto {
|
export class StartJobDto {
|
||||||
@@ -9,4 +9,13 @@ export class StartJobDto {
|
|||||||
@IsOptional()
|
@IsOptional()
|
||||||
@IsString()
|
@IsString()
|
||||||
file?: string;
|
file?: string;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* REIMPORT only: proceed even though the rebuild deletes rows that exist only
|
||||||
|
* in the platform. Off by default, so the guard in run_all.py stops the job
|
||||||
|
* and lists what would be lost rather than the operator finding out after.
|
||||||
|
*/
|
||||||
|
@IsOptional()
|
||||||
|
@IsBoolean()
|
||||||
|
forceFull?: boolean;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ export const SETTING_KEYS = {
|
|||||||
scheduleServicios: "notification.schedule.servicios",
|
scheduleServicios: "notification.schedule.servicios",
|
||||||
/** JSON cadence of the automatic pólizas renewal sweep. */
|
/** JSON cadence of the automatic pólizas renewal sweep. */
|
||||||
schedulePolizas: "notification.schedule.polizas",
|
schedulePolizas: "notification.schedule.polizas",
|
||||||
|
/** Whether the NUMid allocator may reuse empty portal ids. */
|
||||||
|
numidRecycleEmpty: "numid.recycleEmpty",
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
/** Where a resolved value came from. Shown in the UI. */
|
/** Where a resolved value came from. Shown in the UI. */
|
||||||
@@ -169,6 +171,39 @@ export class SettingsService {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether the NUMid allocator may reuse empty portal ids instead of only
|
||||||
|
* issuing new ones.
|
||||||
|
*
|
||||||
|
* Defaults to OFF, and the default is the safety property rather than a
|
||||||
|
* preference: while Access remains the utilities master, every reusable id
|
||||||
|
* still exists in DATGRAL, and a `--sync` migration run reassigns the ref back
|
||||||
|
* to its Access owner (transform_customers.py:327). Recycling before utilities
|
||||||
|
* cuts over therefore hands out ids that quietly stop working. No env rung —
|
||||||
|
* this has never been an environment variable and should be flipped
|
||||||
|
* deliberately, in the UI, by someone who knows the cutover happened.
|
||||||
|
*/
|
||||||
|
async numidRecycleEmpty(): Promise<ResolvedSetting<boolean>> {
|
||||||
|
const row = await this.read(SETTING_KEYS.numidRecycleEmpty);
|
||||||
|
if (row) {
|
||||||
|
return {
|
||||||
|
value: row.value === "true",
|
||||||
|
source: "db",
|
||||||
|
updatedAt: row.updatedAt,
|
||||||
|
updatedById: row.updatedById,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { value: false, source: "default", updatedAt: null, updatedById: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
async setNumidRecycleEmpty(
|
||||||
|
enabled: boolean,
|
||||||
|
userId: string,
|
||||||
|
): Promise<ResolvedSetting<boolean>> {
|
||||||
|
await this.write(SETTING_KEYS.numidRecycleEmpty, String(enabled), userId);
|
||||||
|
return this.numidRecycleEmpty();
|
||||||
|
}
|
||||||
|
|
||||||
private read(key: string) {
|
private read(key: string) {
|
||||||
return this.prisma.appSetting.findUnique({ where: { key } });
|
return this.prisma.appSetting.findUnique({ where: { key } });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@jorgecuadros/web",
|
"name": "@jorgecuadros/web",
|
||||||
"version": "1.0.14",
|
"version": "1.0.17",
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "next dev -p 4500",
|
"dev": "next dev -p 4500",
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import { ContextReports } from "@/components/ContextReports";
|
|||||||
import {
|
import {
|
||||||
archiveCustomer,
|
archiveCustomer,
|
||||||
getCustomer,
|
getCustomer,
|
||||||
|
grantPortalAccess,
|
||||||
policyDocumentDownloadUrl,
|
policyDocumentDownloadUrl,
|
||||||
propertyDocumentDownloadUrl,
|
propertyDocumentDownloadUrl,
|
||||||
restoreCustomer,
|
restoreCustomer,
|
||||||
@@ -151,9 +152,43 @@ function CustomerActions({
|
|||||||
}) {
|
}) {
|
||||||
const canEdit = useCan("customer:update");
|
const canEdit = useCan("customer:update");
|
||||||
const canDelete = useCan("customer:delete");
|
const canDelete = useCan("customer:delete");
|
||||||
|
const canGrantPortal = useCan("customer:portal-access");
|
||||||
const [busy, setBusy] = useState(false);
|
const [busy, setBusy] = useState(false);
|
||||||
const archived = customer.archivedAt != null;
|
const archived = customer.archivedAt != null;
|
||||||
|
|
||||||
|
// The portal NUMid is a legacy ref, not a column: (utilities, DATGRAL) is the
|
||||||
|
// "Security Number" my.jorgecuadros.com asks for. An insurance ref is a
|
||||||
|
// different id space entirely and does not let anyone log in, so both columns
|
||||||
|
// are checked — matching on sourceTable alone would hide the button from
|
||||||
|
// customers who cannot actually reach the portal.
|
||||||
|
const hasPortalId = customer.legacyRefs.some(
|
||||||
|
(r) => r.sourceSystem === "utilities" && r.sourceTable === "DATGRAL",
|
||||||
|
);
|
||||||
|
|
||||||
|
async function grantPortal() {
|
||||||
|
if (
|
||||||
|
!window.confirm(
|
||||||
|
"¿Asignar un número de portal a este cliente? Con él podrá entrar a " +
|
||||||
|
"my.jorgecuadros.com.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return;
|
||||||
|
setBusy(true);
|
||||||
|
try {
|
||||||
|
const { numid, origin } = await grantPortalAccess(customer.id);
|
||||||
|
window.alert(
|
||||||
|
origin === "existing"
|
||||||
|
? `Este cliente ya tenía el número de portal ${numid}.`
|
||||||
|
: `Número de portal asignado: ${numid}.`,
|
||||||
|
);
|
||||||
|
onChange();
|
||||||
|
} catch (e) {
|
||||||
|
window.alert((e as Error)?.message ?? "No se pudo completar la acción.");
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function toggleArchive() {
|
async function toggleArchive() {
|
||||||
const verb = archived ? "restaurar" : "archivar";
|
const verb = archived ? "restaurar" : "archivar";
|
||||||
if (!window.confirm(`¿Seguro que desea ${verb} este cliente?`)) return;
|
if (!window.confirm(`¿Seguro que desea ${verb} este cliente?`)) return;
|
||||||
@@ -169,11 +204,23 @@ function CustomerActions({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!canEdit && !canDelete) return null;
|
const showPortal = canGrantPortal && !hasPortalId && !archived;
|
||||||
|
if (!canEdit && !canDelete && !showPortal) return null;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="row-actions">
|
<div className="row-actions">
|
||||||
{archived && <span className="badge badge-negative">Archivado</span>}
|
{archived && <span className="badge badge-negative">Archivado</span>}
|
||||||
|
{showPortal && (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="btn btn-outline"
|
||||||
|
onClick={grantPortal}
|
||||||
|
disabled={busy}
|
||||||
|
title="Asigna el número que el cliente usa para entrar al portal"
|
||||||
|
>
|
||||||
|
Habilitar acceso al portal
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
{canEdit && (
|
{canEdit && (
|
||||||
<Link href={`/clientes/${customer.id}/editar`} className="btn btn-outline">
|
<Link href={`/clientes/${customer.id}/editar`} className="btn btn-outline">
|
||||||
Editar
|
Editar
|
||||||
|
|||||||
@@ -8,19 +8,18 @@ export const metadata = {
|
|||||||
"Plataforma interna unificada de clientes, servicios y seguros.",
|
"Plataforma interna unificada de clientes, servicios y seguros.",
|
||||||
};
|
};
|
||||||
|
|
||||||
// The browser talks to the API cross-origin, so it needs the API URL at
|
// API_ORIGIN is an OPTIONAL override, read here on the server per request and
|
||||||
// runtime. NEXT_PUBLIC_* would bake it at build time (one URL per image); we
|
// injected as window.__API_ORIGIN__ (see lib/api.ts). NEXT_PUBLIC_* would bake
|
||||||
// want the URL to come from the deploy .env instead. So read it here on the
|
// it at build time (one URL per image); reading it here keeps one image usable
|
||||||
// server per request and inject it as window.__API_ORIGIN__ (see lib/api.ts).
|
// anywhere. Left unset — the normal case — this injects the empty string and
|
||||||
// force-dynamic guarantees process.env is read at request time, never baked
|
// lib/api.ts derives the origin from window.location instead, so the app
|
||||||
// into a static prerender.
|
// follows the server when it moves without an env edit. force-dynamic
|
||||||
|
// guarantees process.env is read at request time, never baked into a static
|
||||||
|
// prerender.
|
||||||
export const dynamic = "force-dynamic";
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
export default function RootLayout({ children }: { children: ReactNode }) {
|
export default function RootLayout({ children }: { children: ReactNode }) {
|
||||||
const apiOrigin =
|
const apiOrigin = process.env.API_ORIGIN ?? "";
|
||||||
process.env.API_ORIGIN ??
|
|
||||||
process.env.NEXT_PUBLIC_API_ORIGIN ??
|
|
||||||
"http://localhost:3001";
|
|
||||||
// Same reason as the API origin: read on the server per request so the built
|
// Same reason as the API origin: read on the server per request so the built
|
||||||
// image is not pinned to one build identity in its client bundle.
|
// image is not pinned to one build identity in its client bundle.
|
||||||
const build = readBuildInfoFromEnv();
|
const build = readBuildInfoFromEnv();
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ import {
|
|||||||
listOpsJobs,
|
listOpsJobs,
|
||||||
startOpsJob,
|
startOpsJob,
|
||||||
uploadIngest,
|
uploadIngest,
|
||||||
|
verifyReplication,
|
||||||
} from "@/lib/api";
|
} from "@/lib/api";
|
||||||
import type { UploadProgress } from "@/lib/api";
|
import type { UploadProgress } from "@/lib/api";
|
||||||
import type {
|
import type {
|
||||||
@@ -28,7 +29,9 @@ import type {
|
|||||||
IngestFile,
|
IngestFile,
|
||||||
OpsJob,
|
OpsJob,
|
||||||
OpsJobKind,
|
OpsJobKind,
|
||||||
|
GtidDrift,
|
||||||
ReplicationStatus,
|
ReplicationStatus,
|
||||||
|
VerifyResult,
|
||||||
} from "@/lib/types";
|
} from "@/lib/types";
|
||||||
|
|
||||||
const INGEST_MAX_BYTES = 2 * 1024 * 1024 * 1024;
|
const INGEST_MAX_BYTES = 2 * 1024 * 1024 * 1024;
|
||||||
@@ -59,6 +62,7 @@ function Operaciones() {
|
|||||||
const [notice, setNotice] = useState<string | null>(null);
|
const [notice, setNotice] = useState<string | null>(null);
|
||||||
const [confirm, setConfirm] = useState<ConfirmState>(null);
|
const [confirm, setConfirm] = useState<ConfirmState>(null);
|
||||||
const [confirmText, setConfirmText] = useState("");
|
const [confirmText, setConfirmText] = useState("");
|
||||||
|
const [forceFull, setForceFull] = useState(false);
|
||||||
const [uploading, setUploading] = useState<string | null>(null);
|
const [uploading, setUploading] = useState<string | null>(null);
|
||||||
const [progress, setProgress] = useState<UploadProgress | null>(null);
|
const [progress, setProgress] = useState<UploadProgress | null>(null);
|
||||||
const [starting, setStarting] = useState(false);
|
const [starting, setStarting] = useState(false);
|
||||||
@@ -159,12 +163,12 @@ function Operaciones() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function start(kind: OpsJobKind, file?: string) {
|
async function start(kind: OpsJobKind, file?: string, force?: boolean) {
|
||||||
setError(null);
|
setError(null);
|
||||||
setNotice(null);
|
setNotice(null);
|
||||||
setStarting(true);
|
setStarting(true);
|
||||||
try {
|
try {
|
||||||
const job = await startOpsJob(kind, file);
|
const job = await startOpsJob(kind, file, force);
|
||||||
setActiveJob(job);
|
setActiveJob(job);
|
||||||
setJobs((prev) => (prev ? [job, ...prev] : [job]));
|
setJobs((prev) => (prev ? [job, ...prev] : [job]));
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -177,6 +181,9 @@ function Operaciones() {
|
|||||||
function askConfirm(state: ConfirmState) {
|
function askConfirm(state: ConfirmState) {
|
||||||
setConfirm(state);
|
setConfirm(state);
|
||||||
setConfirmText("");
|
setConfirmText("");
|
||||||
|
// Always re-armed: ticking "delete native rows" once must not carry into
|
||||||
|
// the next reimport.
|
||||||
|
setForceFull(false);
|
||||||
setError(null);
|
setError(null);
|
||||||
setNotice(null);
|
setNotice(null);
|
||||||
}
|
}
|
||||||
@@ -185,7 +192,7 @@ function Operaciones() {
|
|||||||
if (!confirm) return;
|
if (!confirm) return;
|
||||||
const c = confirm;
|
const c = confirm;
|
||||||
setConfirm(null);
|
setConfirm(null);
|
||||||
if (c.kind === "REIMPORT") await start("REIMPORT");
|
if (c.kind === "REIMPORT") await start("REIMPORT", undefined, forceFull);
|
||||||
else if (c.kind === "SYNC") await start("SYNC");
|
else if (c.kind === "SYNC") await start("SYNC");
|
||||||
else await start("RESTORE", c.file);
|
else await start("RESTORE", c.file);
|
||||||
}
|
}
|
||||||
@@ -478,11 +485,24 @@ function Operaciones() {
|
|||||||
</h2>
|
</h2>
|
||||||
<p className="inline-form-note">
|
<p className="inline-form-note">
|
||||||
{confirm.kind === "REIMPORT"
|
{confirm.kind === "REIMPORT"
|
||||||
? "Esto BORRA todos los datos actuales (incluidos los capturados a mano) y reconstruye desde los archivos de ingesta. Se creará un respaldo previo automático."
|
? "Esto BORRA todos los datos actuales y reconstruye desde los archivos de ingesta. Se creará un respaldo previo automático. Si la base contiene registros que sólo existen en la plataforma (clientes creados aquí, números de portal asignados, pólizas capturadas por OCR, movimientos capturados), la operación se detiene y los enumera sin tocar nada."
|
||||||
: confirm.kind === "SYNC"
|
: confirm.kind === "SYNC"
|
||||||
? "Se creará un respaldo previo automático. Luego se importarán al sistema los registros nuevos del legado y se eliminarán los del legado que ya no aparezcan en los archivos de ingesta. Los datos capturados a mano NO se borran."
|
? "Se creará un respaldo previo automático. Luego se importarán al sistema los registros nuevos del legado y se eliminarán los del legado que ya no aparezcan en los archivos de ingesta. Los datos capturados a mano NO se borran."
|
||||||
: `Esto sobreescribe la base de datos completa con “${confirm.file}”. Se recomienda crear un respaldo antes.`}
|
: `Esto sobreescribe la base de datos completa con “${confirm.file}”. Se recomienda crear un respaldo antes.`}
|
||||||
</p>
|
</p>
|
||||||
|
{confirm.kind === "REIMPORT" && (
|
||||||
|
<label className="inline-form-note" style={{ display: "block" }}>
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={forceFull}
|
||||||
|
onChange={(e) => setForceFull(e.target.checked)}
|
||||||
|
style={{ marginRight: 8 }}
|
||||||
|
/>
|
||||||
|
Borrar también los registros que sólo existen en la plataforma
|
||||||
|
(ignorar la verificación). Sólo marque esto si de verdad quiere
|
||||||
|
perderlos.
|
||||||
|
</label>
|
||||||
|
)}
|
||||||
<label className="field">
|
<label className="field">
|
||||||
<span className="field-label">Escriba CONFIRMAR para continuar</span>
|
<span className="field-label">Escriba CONFIRMAR para continuar</span>
|
||||||
<input
|
<input
|
||||||
@@ -614,6 +634,9 @@ function OpTile({
|
|||||||
function ReplicationCard() {
|
function ReplicationCard() {
|
||||||
const [status, setStatus] = useState<ReplicationStatus | null>(null);
|
const [status, setStatus] = useState<ReplicationStatus | null>(null);
|
||||||
const [failed, setFailed] = useState(false);
|
const [failed, setFailed] = useState(false);
|
||||||
|
const [verify, setVerify] = useState<VerifyResult | null>(null);
|
||||||
|
const [verifying, setVerifying] = useState(false);
|
||||||
|
const [verifyError, setVerifyError] = useState<string | null>(null);
|
||||||
|
|
||||||
const load = useCallback(() => {
|
const load = useCallback(() => {
|
||||||
getReplicationStatus()
|
getReplicationStatus()
|
||||||
@@ -624,6 +647,17 @@ function ReplicationCard() {
|
|||||||
.catch(() => setFailed(true));
|
.catch(() => setFailed(true));
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const runVerify = useCallback(() => {
|
||||||
|
setVerifying(true);
|
||||||
|
setVerifyError(null);
|
||||||
|
verifyReplication()
|
||||||
|
.then(setVerify)
|
||||||
|
.catch((e: unknown) =>
|
||||||
|
setVerifyError(e instanceof Error ? e.message : "No se pudo comparar."),
|
||||||
|
)
|
||||||
|
.finally(() => setVerifying(false));
|
||||||
|
}, []);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
load();
|
load();
|
||||||
const t = setInterval(load, 30_000);
|
const t = setInterval(load, 30_000);
|
||||||
@@ -687,14 +721,144 @@ function ReplicationCard() {
|
|||||||
value={status.secondsBehind === null ? "sin dato" : `${status.secondsBehind} s`}
|
value={status.secondsBehind === null ? "sin dato" : `${status.secondsBehind} s`}
|
||||||
/>
|
/>
|
||||||
<KV label="Pendiente de aplicar" value={backlogLabel(status.apply)} />
|
<KV label="Pendiente de aplicar" value={backlogLabel(status.apply)} />
|
||||||
|
<KV label="Diferencia con el maestro" value={driftLabel(status.drift)} />
|
||||||
<KV label="Consultado" value={formatDateTime(status.checkedAt)} />
|
<KV label="Consultado" value={formatDateTime(status.checkedAt)} />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<ApplyProgressBar apply={status.apply} />
|
<ApplyProgressBar apply={status.apply} />
|
||||||
|
|
||||||
|
{/* Only worth showing when it is not zero, and even then as a note rather
|
||||||
|
than a warning: these are the seed load's own transactions, and they
|
||||||
|
are inert until someone tries to promote this box. */}
|
||||||
|
{status.drift !== null && status.drift.localTransactions > 0 && (
|
||||||
|
<p className="inline-form-note" style={{ marginTop: 12 }}>
|
||||||
|
La réplica tiene {status.drift.localTransactions.toLocaleString("es-MX")} transacciones
|
||||||
|
propias (de la carga inicial). No se propagan y no afectan la lectura; sólo importarían
|
||||||
|
si este servidor pasara a ser maestro.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<VerifyPanel
|
||||||
|
result={verify}
|
||||||
|
running={verifying}
|
||||||
|
error={verifyError}
|
||||||
|
onRun={runVerify}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Row-by-row comparison against the master, on demand.
|
||||||
|
*
|
||||||
|
* Separate from the polled fields because it costs a full scan of both servers.
|
||||||
|
* It is the only check here that can catch a row changed on the replica by
|
||||||
|
* something other than replication — the GTID and lag figures would both still
|
||||||
|
* read perfectly healthy in that case.
|
||||||
|
*/
|
||||||
|
function VerifyPanel({
|
||||||
|
result,
|
||||||
|
running,
|
||||||
|
error,
|
||||||
|
onRun,
|
||||||
|
}: {
|
||||||
|
result: VerifyResult | null;
|
||||||
|
running: boolean;
|
||||||
|
error: string | null;
|
||||||
|
onRun: () => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<div style={{ marginTop: 16, borderTop: "1px solid var(--border)", paddingTop: 12 }}>
|
||||||
|
<div className="row-actions" style={{ justifyContent: "space-between" }}>
|
||||||
|
<span className="inline-form-note" style={{ margin: 0 }}>
|
||||||
|
Compara fila por fila las 8 tablas que lee el sitio de clientes. Recorre ambos
|
||||||
|
servidores por completo, así que tarda.
|
||||||
|
</span>
|
||||||
|
<button className="btn btn-ghost" type="button" onClick={onRun} disabled={running}>
|
||||||
|
{running ? "Comparando…" : "Comparar con el maestro"}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{error && (
|
||||||
|
<div className="state-box state-error" style={{ marginTop: 12 }}>
|
||||||
|
{error}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{result?.problem && (
|
||||||
|
<div className="state-box state-error" style={{ marginTop: 12 }}>
|
||||||
|
{result.problem}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{result && !result.problem && (
|
||||||
|
<>
|
||||||
|
<p style={{ marginTop: 12, marginBottom: 8 }}>
|
||||||
|
<span className={`badge ${result.identical ? "badge-positive" : "badge-negative"}`}>
|
||||||
|
{result.identical ? "Idénticas" : "Hay diferencias"}
|
||||||
|
</span>{" "}
|
||||||
|
<span className="inline-form-note">
|
||||||
|
{formatDateTime(result.checkedAt)} · {(result.elapsedMs / 1000).toFixed(1)} s
|
||||||
|
</span>
|
||||||
|
</p>
|
||||||
|
<div style={{ overflowX: "auto" }}>
|
||||||
|
<table className="data-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Tabla</th>
|
||||||
|
<th style={{ textAlign: "right" }}>Maestro</th>
|
||||||
|
<th style={{ textAlign: "right" }}>Réplica</th>
|
||||||
|
<th>Estado</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{result.tables.map((t) => (
|
||||||
|
<tr key={t.table}>
|
||||||
|
<td>{t.table}</td>
|
||||||
|
{/* -1 is the sentinel for "that server did not answer for
|
||||||
|
this table", which is not the same as zero rows. */}
|
||||||
|
<td style={{ textAlign: "right" }}>
|
||||||
|
{t.masterRows < 0 ? "—" : t.masterRows.toLocaleString("es-MX")}
|
||||||
|
</td>
|
||||||
|
<td style={{ textAlign: "right" }}>
|
||||||
|
{t.replicaRows < 0 ? "—" : t.replicaRows.toLocaleString("es-MX")}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<span className={`badge ${t.matches ? "badge-positive" : "badge-negative"}`}>
|
||||||
|
{t.matches
|
||||||
|
? "igual"
|
||||||
|
: t.masterRows !== t.replicaRows
|
||||||
|
? "difieren en filas"
|
||||||
|
: "difieren en contenido"}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transactions the master has executed that the replica has not.
|
||||||
|
*
|
||||||
|
* Rendered as its own line rather than folded into the lag figure because the
|
||||||
|
* two disagree in exactly the case that matters: a disconnected I/O thread
|
||||||
|
* reports 0 seconds of lag (no event has arrived to be late) while this number
|
||||||
|
* climbs.
|
||||||
|
*/
|
||||||
|
function driftLabel(drift: GtidDrift | null): string {
|
||||||
|
// Null means the master could not be reached. Saying "al día" here would be a
|
||||||
|
// lie of the worst kind — it is the reading a broken check produces.
|
||||||
|
if (drift === null) return "sin dato";
|
||||||
|
if (drift.missingTransactions === 0) return "al día";
|
||||||
|
return `${drift.missingTransactions.toLocaleString("es-MX")} transacciones atrás`;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Bytes the replica has fetched but not yet applied.
|
* Bytes the replica has fetched but not yet applied.
|
||||||
*
|
*
|
||||||
|
|||||||
+51
-7
@@ -60,6 +60,7 @@ import type {
|
|||||||
OpsJob,
|
OpsJob,
|
||||||
OpsJobKind,
|
OpsJobKind,
|
||||||
ReplicationStatus,
|
ReplicationStatus,
|
||||||
|
VerifyResult,
|
||||||
IngestFile,
|
IngestFile,
|
||||||
BackupFile,
|
BackupFile,
|
||||||
PropertyDetail,
|
PropertyDetail,
|
||||||
@@ -81,15 +82,26 @@ import type {
|
|||||||
UserRow,
|
UserRow,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
|
|
||||||
// Resolve the API origin at runtime, not build time. In the browser it comes
|
// Resolve the API origin at runtime, not build time — so one built image serves
|
||||||
// from window.__API_ORIGIN__, injected server-side by the root layout from the
|
// any deployment and the app follows the box when it moves (tailnet today,
|
||||||
// deploy .env (API_ORIGIN) — so one built image serves any deployment. On the
|
// 192.168.1.x office LAN later) with no config change.
|
||||||
// server (SSR) read process.env directly. NEXT_PUBLIC_API_ORIGIN stays as the
|
//
|
||||||
// dev/build fallback.
|
// In the browser, derive the origin from the page's own location, the way a PHP
|
||||||
|
// app would. An explicit API_ORIGIN (injected as window.__API_ORIGIN__ by the
|
||||||
|
// root layout) still wins when a deployment genuinely splits the two hosts.
|
||||||
|
// On the server (SSR) read process.env directly — a derived origin is
|
||||||
|
// browser-only, and "/api" is not fetchable server-side.
|
||||||
function resolveApiOrigin(): string {
|
function resolveApiOrigin(): string {
|
||||||
if (typeof window !== "undefined") {
|
if (typeof window !== "undefined") {
|
||||||
const injected = (window as { __API_ORIGIN__?: string }).__API_ORIGIN__;
|
const injected = (window as { __API_ORIGIN__?: string }).__API_ORIGIN__;
|
||||||
if (injected) return injected;
|
if (injected) return injected;
|
||||||
|
const { protocol, hostname } = window.location;
|
||||||
|
// Over TLS the API must share the page's origin or the browser blocks the
|
||||||
|
// call as mixed active content. The reverse proxy maps /api to the API.
|
||||||
|
if (protocol === "https:") return "/api";
|
||||||
|
// Plain HTTP: same host, API port. 3001 is the port the API container
|
||||||
|
// publishes everywhere (deploy/galactus/jorgecuadros-app.compose.yml).
|
||||||
|
return `http://${hostname}:3001`;
|
||||||
}
|
}
|
||||||
return (
|
return (
|
||||||
process.env.API_ORIGIN ??
|
process.env.API_ORIGIN ??
|
||||||
@@ -232,6 +244,20 @@ export function restoreCustomer(id: string): Promise<CustomerDetail> {
|
|||||||
return apiFetch<CustomerDetail>(`/customers/${id}/restore`, { method: "POST" });
|
return apiFetch<CustomerDetail>(`/customers/${id}/restore`, { method: "POST" });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface NumidAllocation {
|
||||||
|
numid: string;
|
||||||
|
/** "existing" when the customer already had one — the call is idempotent. */
|
||||||
|
origin: "existing" | "new" | "recycled";
|
||||||
|
previousCustomerId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Give a customer the portal NUMid they log in to my.jorgecuadros.com with. */
|
||||||
|
export function grantPortalAccess(id: string): Promise<NumidAllocation> {
|
||||||
|
return apiFetch<NumidAllocation>(`/customers/${id}/portal-access`, {
|
||||||
|
method: "POST",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
/* ------------------------------------------------------ Policies module */
|
/* ------------------------------------------------------ Policies module */
|
||||||
|
|
||||||
/** Renewal horizon in days, shared by the list, stats and detail calls so the
|
/** Renewal horizon in days, shared by the list, stats and detail calls so the
|
||||||
@@ -950,6 +976,18 @@ export function getReplicationStatus(): Promise<ReplicationStatus> {
|
|||||||
return apiFetch<ReplicationStatus>("/ops/replication");
|
return apiFetch<ReplicationStatus>("/ops/replication");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compare every customer-visible table against the master, row by row.
|
||||||
|
*
|
||||||
|
* Slow by nature — it is a full scan of both servers — so it is a button, not
|
||||||
|
* part of the poll. Answers the question replication status cannot: GTIDs prove
|
||||||
|
* the replica applied everything the master sent, not that nothing else changed
|
||||||
|
* the rows here.
|
||||||
|
*/
|
||||||
|
export function verifyReplication(): Promise<VerifyResult> {
|
||||||
|
return apiFetch<VerifyResult>("/ops/replication/verify", { method: "POST" });
|
||||||
|
}
|
||||||
|
|
||||||
export function listOpsJobs(): Promise<OpsJob[]> {
|
export function listOpsJobs(): Promise<OpsJob[]> {
|
||||||
return apiFetch<OpsJob[]>("/ops/jobs");
|
return apiFetch<OpsJob[]>("/ops/jobs");
|
||||||
}
|
}
|
||||||
@@ -959,10 +997,16 @@ export function getOpsJob(id: string): Promise<OpsJob> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Start a mutating op. `file` is required for RESTORE. 409 if one is running. */
|
/** Start a mutating op. `file` is required for RESTORE. 409 if one is running. */
|
||||||
export function startOpsJob(kind: OpsJobKind, file?: string): Promise<OpsJob> {
|
/** `forceFull` applies to REIMPORT only: proceed even though the rebuild
|
||||||
|
* deletes rows that exist only in the platform. */
|
||||||
|
export function startOpsJob(
|
||||||
|
kind: OpsJobKind,
|
||||||
|
file?: string,
|
||||||
|
forceFull?: boolean,
|
||||||
|
): Promise<OpsJob> {
|
||||||
return apiFetch<OpsJob>("/ops/jobs", {
|
return apiFetch<OpsJob>("/ops/jobs", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
body: JSON.stringify({ kind, file }),
|
body: JSON.stringify({ kind, file, forceFull }),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ export type Ability =
|
|||||||
| "customer:create"
|
| "customer:create"
|
||||||
| "customer:update"
|
| "customer:update"
|
||||||
| "customer:delete"
|
| "customer:delete"
|
||||||
|
| "customer:portal-access"
|
||||||
| "policy:create"
|
| "policy:create"
|
||||||
| "policy:update"
|
| "policy:update"
|
||||||
| "policy:delete"
|
| "policy:delete"
|
||||||
@@ -100,10 +101,52 @@ export interface ReplicationStatus {
|
|||||||
lastSqlError: string | null;
|
lastSqlError: string | null;
|
||||||
sourceHost: string | null;
|
sourceHost: string | null;
|
||||||
apply: ApplyProgress | null;
|
apply: ApplyProgress | null;
|
||||||
|
drift: GtidDrift | null;
|
||||||
problem: string | null;
|
problem: string | null;
|
||||||
checkedAt: string;
|
checkedAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Executed-history gap between master and replica, in transactions.
|
||||||
|
*
|
||||||
|
* The only field on the card that is not self-reported by the replica, and the
|
||||||
|
* only one that catches a silently disconnected I/O thread: with no incoming
|
||||||
|
* events, `secondsBehind` reads 0 because there is nothing to measure staleness
|
||||||
|
* against, so a dead link looks perfectly current. This number grows instead.
|
||||||
|
*
|
||||||
|
* Null when the master could not be reached — "unknown" must not render as
|
||||||
|
* "identical".
|
||||||
|
*/
|
||||||
|
export interface GtidDrift {
|
||||||
|
missingTransactions: number;
|
||||||
|
missingGtidSet: string | null;
|
||||||
|
/**
|
||||||
|
* Transactions written on the replica under its own server UUID, which exist
|
||||||
|
* nowhere on the master. Non-zero is expected — restoring the seed dump
|
||||||
|
* executed its statements locally — and harmless while nothing replicates
|
||||||
|
* from this node.
|
||||||
|
*/
|
||||||
|
localTransactions: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One table compared on both sides of the link. */
|
||||||
|
export interface TableFingerprint {
|
||||||
|
table: string;
|
||||||
|
masterRows: number;
|
||||||
|
replicaRows: number;
|
||||||
|
masterChecksum: string;
|
||||||
|
replicaChecksum: string;
|
||||||
|
matches: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VerifyResult {
|
||||||
|
identical: boolean;
|
||||||
|
tables: TableFingerprint[];
|
||||||
|
problem: string | null;
|
||||||
|
checkedAt: string;
|
||||||
|
elapsedMs: number;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Relay-log apply progress, in source binlog bytes.
|
* Relay-log apply progress, in source binlog bytes.
|
||||||
*
|
*
|
||||||
|
|||||||
@@ -132,8 +132,12 @@ services:
|
|||||||
dns_search:
|
dns_search:
|
||||||
- ${TAILNET_SUFFIX:-tail01aa2.ts.net}
|
- ${TAILNET_SUFFIX:-tail01aa2.ts.net}
|
||||||
environment:
|
environment:
|
||||||
# Public API URL the browser calls (injected at runtime, see layout.tsx).
|
# OPTIONAL override of the API URL the browser calls (injected at runtime,
|
||||||
API_ORIGIN: ${API_ORIGIN:?API_ORIGIN must be set}
|
# see layout.tsx). Leave it unset: the browser then derives the origin
|
||||||
|
# from the page it loaded — same host on port 3001 over plain HTTP, or
|
||||||
|
# /api behind a TLS-terminating proxy. Set it only when the API really
|
||||||
|
# lives on a different host than the web app.
|
||||||
|
API_ORIGIN: ${API_ORIGIN:-}
|
||||||
ports:
|
ports:
|
||||||
- "${WEB_PORT:-3000}:3000"
|
- "${WEB_PORT:-3000}:3000"
|
||||||
depends_on:
|
depends_on:
|
||||||
|
|||||||
@@ -8,10 +8,21 @@
|
|||||||
APP_TAG=latest
|
APP_TAG=latest
|
||||||
|
|
||||||
# --- Public URLs (what the end user's BROWSER hits) ---------------------------
|
# --- Public URLs (what the end user's BROWSER hits) ---------------------------
|
||||||
# API_ORIGIN is injected into the web app at runtime and used for browser fetches
|
# API_ORIGIN is OPTIONAL and normally left unset. The browser derives the API
|
||||||
# + document download links, so it must be browser-reachable (not swarm-internal).
|
# origin from the page it loaded (apps/web/src/lib/api.ts): same host on port
|
||||||
# WEB_ORIGIN is the web app's own public origin; the API allows it via CORS.
|
# 3001 over plain HTTP, or the same-origin /api path when the page is served
|
||||||
API_ORIGIN=http://192.168.4.212:3001
|
# over https by a TLS-terminating proxy that maps /api to the API. That is what
|
||||||
|
# lets the same deployment move — tailnet, office LAN, demo domain — untouched.
|
||||||
|
# Set it only when the API genuinely lives on a different host than the web app;
|
||||||
|
# it is used for browser fetches AND document download links, so it must be
|
||||||
|
# browser-reachable (never a swarm-internal name).
|
||||||
|
#API_ORIGIN=http://192.168.4.212:3001
|
||||||
|
#
|
||||||
|
# WEB_ORIGIN is the list of public origins the web app is reached under; the API
|
||||||
|
# allows them via CORS. COMMA-SEPARATED — one deployment is reachable under
|
||||||
|
# several origins (LAN IP, tailnet name, demo domain) and a credentialed fetch
|
||||||
|
# from an origin missing here gets no CORS headers and fails. A same-origin
|
||||||
|
# setup (web + API behind one proxy) never hits CORS at all.
|
||||||
WEB_ORIGIN=http://192.168.4.212:3000
|
WEB_ORIGIN=http://192.168.4.212:3000
|
||||||
|
|
||||||
# Published ports on the swarm host.
|
# Published ports on the swarm host.
|
||||||
|
|||||||
@@ -92,8 +92,12 @@ services:
|
|||||||
labels:
|
labels:
|
||||||
io.jorgecuadros.role: "web"
|
io.jorgecuadros.role: "web"
|
||||||
environment:
|
environment:
|
||||||
# Public API URL the browser calls (injected at runtime, see layout.tsx).
|
# OPTIONAL override of the API URL the browser calls (injected at runtime,
|
||||||
API_ORIGIN: ${API_ORIGIN:?API_ORIGIN must be set}
|
# see layout.tsx). Leave it unset: the browser then derives the origin
|
||||||
|
# from the page it loaded — same host on port 3001 over plain HTTP, or
|
||||||
|
# /api behind a TLS-terminating proxy. Set it only when the API really
|
||||||
|
# lives on a different host than the web app.
|
||||||
|
API_ORIGIN: ${API_ORIGIN:-}
|
||||||
ports:
|
ports:
|
||||||
- target: 3000
|
- target: 3000
|
||||||
published: ${WEB_PORT:-3000}
|
published: ${WEB_PORT:-3000}
|
||||||
|
|||||||
+28
-1
@@ -83,7 +83,7 @@ guess.
|
|||||||
| 1.3 | Carrier API **direction**: outbound quote/issue (ANA supports today) or inbound portfolio sync (no evidence either carrier offers it) | whether §4 is buildable at all | INSURANCE §4 |
|
| 1.3 | Carrier API **direction**: outbound quote/issue (ANA supports today) or inbound portfolio sync (no evidence either carrier offers it) | whether §4 is buildable at all | INSURANCE §4 |
|
||||||
| 1.4 | CFE amount: the rounded barcode figure (`$268`, what is paid at the window) or the exact breakdown total (`$268.88`) | the parser currently takes the barcode | STATEMENT_OCR / RECEIPT §2 |
|
| 1.4 | CFE amount: the rounded barcode figure (`$268`, what is paid at the window) or the exact breakdown total (`$268.88`) | the parser currently takes the barcode | STATEMENT_OCR / RECEIPT §2 |
|
||||||
| 1.5 | The Seguros USD bank's name, currency and details | multi-bank is built; that account does not exist yet | RECEIPT §3 |
|
| 1.5 | The Seguros USD bank's name, currency and details | multi-bank is built; that account does not exist yet | RECEIPT §3 |
|
||||||
| 1.6 | Recycling triggers — exact "1 year inactive" / "cancelled" definitions, and whether recycling ever means true data purge | §4 recycling | RECEIPT §4 |
|
| 1.6 | Whether recycling ever means a true data purge. The *triggers* are now settled and built (see §5 "NUMid allocation"); what is still open is whether a recycled id's old rows are ever deleted rather than left attached to the previous customer | nothing — the allocator ships without a purge | RECEIPT §4 |
|
||||||
| 1.7 | Notice body in Spanish or English | `Customer` carries no language preference | INSURANCE §1 |
|
| 1.7 | Notice body in Spanish or English | `Customer` carries no language preference | INSURANCE §1 |
|
||||||
| 1.8 | How to model `TRASPASOS PAYPAL` — a clearing account, not a customer, carrying −7.03M MXN over 309 movements and therefore topping the adeudo worklist | deliberately not special-cased in code | RESUME §6 |
|
| 1.8 | How to model `TRASPASOS PAYPAL` — a clearing account, not a customer, carrying −7.03M MXN over 309 movements and therefore topping the adeudo worklist | deliberately not special-cased in code | RESUME §6 |
|
||||||
| 1.9 | The 78 policyholders with no email — skip silently or produce a print worklist | recommendation is the worklist | INSURANCE §1 |
|
| 1.9 | The 78 policyholders with no email — skip silently or produce a print worklist | recommendation is the worklist | INSURANCE §1 |
|
||||||
@@ -184,6 +184,33 @@ Each of these is a known, deliberate stopping point rather than a bug.
|
|||||||
- Handwritten folder numbers are deliberately not an input to matching
|
- Handwritten folder numbers are deliberately not an input to matching
|
||||||
(Tesseract read `405` as `205`).
|
(Tesseract read `405` as `205`).
|
||||||
|
|
||||||
|
**NUMid allocation** — `POST /customers/:id/portal-access` assigns the portal
|
||||||
|
"Security Number", on a staff action rather than at create time, because an
|
||||||
|
insurance-only customer has no reason to hold a utilities id.
|
||||||
|
|
||||||
|
- **Recycling is built but switched off.** `numid.recycleEmpty` in `app_settings`
|
||||||
|
defaults to false, and that default is a safety property, not a preference:
|
||||||
|
every reusable id still exists in Access DATGRAL, and a `--sync` migration run
|
||||||
|
upserts refs with `ON DUPLICATE KEY UPDATE customerId`
|
||||||
|
(`transform_customers.py:327`), so an id recycled today is handed back to its
|
||||||
|
Access owner on the next sync and the customer given it loses portal access.
|
||||||
|
**Flip it on after utilities cuts over**, or for ids deleted at the source.
|
||||||
|
- **A full re-import would destroy every natively allocated id — now guarded.**
|
||||||
|
`transform_customers.py:246` truncates `customers` and `customer_legacy_refs`
|
||||||
|
(and the other transforms truncate everything they own), then rebuild from
|
||||||
|
Access alone. `migration/native_guard.py` runs before any of it and refuses
|
||||||
|
when the target holds rows Access has never seen; `run_all.py --force-full`,
|
||||||
|
or the checkbox in the REIMPORT confirm, overrides and deletes them. **`--sync`
|
||||||
|
remains the correct path for any database with native rows** — the guard stops
|
||||||
|
the loss, it does not make full mode preserve anything.
|
||||||
|
- **The empty-id rule exists twice**: enforced in `numid.service.ts`
|
||||||
|
(`EMPTY_NUMID_SQL`) and reported by `scripts/numid-audit.sql`. They agree today
|
||||||
|
(both return 1089, 1094, 1134, 1143 on dev); they are not mechanically kept in
|
||||||
|
step, so change them together.
|
||||||
|
- **No un-assign.** Nothing removes a NUMid once given, and nothing reports which
|
||||||
|
ids were recycled from whom beyond the `customer.portal-access` activity-log
|
||||||
|
entry.
|
||||||
|
|
||||||
**Bank** — the concept→ramo classifier is **won't-build**, not pending.
|
**Bank** — the concept→ramo classifier is **won't-build**, not pending.
|
||||||
`concepto` is a payee name (0 of 22,354 match a category) and TABLA RAMODOS is
|
`concepto` is a payee name (0 of 22,354 match a category) and TABLA RAMODOS is
|
||||||
a property-management expense chart, not the business-line split it was assumed
|
a property-management expense chart, not the business-line split it was assumed
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
"""
|
||||||
|
Refuse a full re-import that would delete platform-native data.
|
||||||
|
|
||||||
|
A full `run_all.py` pass truncates and rebuilds every table it owns from the
|
||||||
|
Access extract:
|
||||||
|
|
||||||
|
transform_customers.py customers, customer_legacy_refs
|
||||||
|
transform_properties.py properties, property_services, service_documents,
|
||||||
|
trust_accounts
|
||||||
|
transform_policies.py policies + installments, vehicles, drivers,
|
||||||
|
beneficiaries, claims, adjusters, policy_types,
|
||||||
|
insurance_providers
|
||||||
|
transform_transactions.py transactions, type_transactions, exchange_rates
|
||||||
|
transform_bank.py bank tables
|
||||||
|
blob_extract.py service_documents, policy_documents
|
||||||
|
|
||||||
|
That was harmless while the platform was a read-only mirror of Access: every
|
||||||
|
row came from the extract, so wiping and rebuilding lost nothing. It stopped
|
||||||
|
being harmless when the platform started minting rows Access has never heard
|
||||||
|
of — portal NUMids from the allocator (apps/api/src/customers/numid.service.ts),
|
||||||
|
customers created in the staff UI, OCR-captured policies, app-booked ledger
|
||||||
|
rows, uploaded documents. None of those come back.
|
||||||
|
|
||||||
|
`--sync` already avoids all of it: it upserts legacy rows against the existing
|
||||||
|
refs and leaves everything else alone. So this guard does not try to teach the
|
||||||
|
full path to preserve anything — it stops the full path when there is something
|
||||||
|
to preserve, and points at the additive one.
|
||||||
|
|
||||||
|
python native_guard.py --env prod # report only, exit 3 if blocking
|
||||||
|
python run_all.py --env prod --stage # runs this first, refuses on 3
|
||||||
|
python run_all.py --env prod --force-full # ignore the guard (deletes them)
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from dbenv import connect
|
||||||
|
|
||||||
|
STG = Path(__file__).parent / "output"
|
||||||
|
|
||||||
|
# Exit code the orchestrator looks for. Distinct from 1 so a connection failure
|
||||||
|
# or a bad query is not silently read as "native rows found".
|
||||||
|
BLOCKED = 3
|
||||||
|
|
||||||
|
# transform_customers.py mints these when a legacy row carries no id of its own.
|
||||||
|
# They are regenerated by every full pass, so they are legacy-owned, not native.
|
||||||
|
SYNTHETIC_REF_PREFIXES = ("rownum_", "insrow_")
|
||||||
|
|
||||||
|
# App-uploaded documents are stored as `<prefix>/<parent>/<uuid>.<ext>`, while
|
||||||
|
# blob_extract writes `<prefix>/<parent>/<stagedtable>_<row>_<col>.<ext>`.
|
||||||
|
# service_documents carries no provenance column, so the key shape is the only
|
||||||
|
# signal available — approximate, and reported as such. Matched in MySQL rather
|
||||||
|
# than in Python so the whole scan stays one round trip per table.
|
||||||
|
UUID_KEY_SQL = "/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\\."
|
||||||
|
|
||||||
|
|
||||||
|
def staged_legacy_ids() -> dict[str, set[str]] | None:
|
||||||
|
"""The ids Access will re-create, per source system.
|
||||||
|
|
||||||
|
None when the staged Parquet is absent — which is not the same as "no legacy
|
||||||
|
ids". Returning an empty set there would mark all 1,171 refs as native and
|
||||||
|
block every run; returning None lets the caller say "cannot verify" instead.
|
||||||
|
"""
|
||||||
|
sources = {"utilities": "stg_utilities", "insurance": "stg_seguros"}
|
||||||
|
out: dict[str, set[str]] = {}
|
||||||
|
for system, folder in sources.items():
|
||||||
|
path = STG / folder / "datgral.parquet"
|
||||||
|
if not path.exists():
|
||||||
|
return None
|
||||||
|
df = pd.read_parquet(path, columns=["num_id"])
|
||||||
|
ids = set()
|
||||||
|
for v in df["num_id"].astype("string"):
|
||||||
|
if v is None or pd.isna(v):
|
||||||
|
continue
|
||||||
|
v = str(v).strip()
|
||||||
|
if v.endswith(".0"): # some numeric ids serialize as "521.0"
|
||||||
|
v = v[:-2]
|
||||||
|
if v and v != "0":
|
||||||
|
ids.add(v)
|
||||||
|
out[system] = ids
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def native_refs(cur, staged: dict[str, set[str]] | None) -> tuple[int, list[str]]:
|
||||||
|
"""Legacy refs with no counterpart in the Access extract.
|
||||||
|
|
||||||
|
This is the check that catches an allocated portal NUMid: the customer holds
|
||||||
|
a perfectly ordinary-looking (utilities, DATGRAL, '1172') ref, so "customer
|
||||||
|
has no refs" does not see it. Only comparing against staging does.
|
||||||
|
"""
|
||||||
|
cur.execute(
|
||||||
|
"SELECT sourceSystem, legacyId FROM customer_legacy_refs ORDER BY sourceSystem, legacyId"
|
||||||
|
)
|
||||||
|
rows = cur.fetchall()
|
||||||
|
if staged is None:
|
||||||
|
return 0, []
|
||||||
|
|
||||||
|
found = []
|
||||||
|
for system, legacy_id in rows:
|
||||||
|
if legacy_id.startswith(SYNTHETIC_REF_PREFIXES):
|
||||||
|
continue
|
||||||
|
known = staged.get(system)
|
||||||
|
# An unknown source system has no extract to compare against, so it
|
||||||
|
# cannot be re-created either — treat it as native rather than ignoring.
|
||||||
|
if known is None or legacy_id not in known:
|
||||||
|
found.append(f"{system}/{legacy_id}")
|
||||||
|
return len(found), found
|
||||||
|
|
||||||
|
|
||||||
|
def scan(conn) -> tuple[list[tuple[str, int, str]], bool]:
|
||||||
|
"""(label, count, detail) per source of native rows, plus whether staging
|
||||||
|
was available to verify the refs."""
|
||||||
|
cur = conn.cursor()
|
||||||
|
staged = staged_legacy_ids()
|
||||||
|
findings: list[tuple[str, int, str]] = []
|
||||||
|
|
||||||
|
ref_count, ref_examples = native_refs(cur, staged)
|
||||||
|
if ref_count:
|
||||||
|
shown = ", ".join(ref_examples[:8])
|
||||||
|
more = f" (+{ref_count - 8} more)" if ref_count > 8 else ""
|
||||||
|
findings.append(("customer_legacy_refs", ref_count, f"{shown}{more}"))
|
||||||
|
|
||||||
|
cur.execute(
|
||||||
|
"SELECT COUNT(*) FROM customers c"
|
||||||
|
" WHERE NOT EXISTS (SELECT 1 FROM customer_legacy_refs r WHERE r.customerId = c.id)"
|
||||||
|
)
|
||||||
|
n = cur.fetchone()[0]
|
||||||
|
if n:
|
||||||
|
findings.append(("customers", n, "created in the staff UI, no legacy ref"))
|
||||||
|
|
||||||
|
# Every transform writes legacyId on what it loads, so a NULL is the app's.
|
||||||
|
for table, detail in (
|
||||||
|
("transactions", "booked in the app (captura, OCR, manual)"),
|
||||||
|
("policies", "created in the app or captured by policy OCR"),
|
||||||
|
("properties", "created in the app"),
|
||||||
|
("vehicles", "created in the app"),
|
||||||
|
("bank_transactions", "booked in the chequera"),
|
||||||
|
):
|
||||||
|
cur.execute(f"SELECT COUNT(*) FROM {table} WHERE legacyId IS NULL")
|
||||||
|
n = cur.fetchone()[0]
|
||||||
|
if n:
|
||||||
|
findings.append((table, n, detail))
|
||||||
|
|
||||||
|
# blob_extract always writes originalColumn; the app never does.
|
||||||
|
cur.execute("SELECT COUNT(*) FROM policy_documents WHERE originalColumn IS NULL")
|
||||||
|
n = cur.fetchone()[0]
|
||||||
|
if n:
|
||||||
|
findings.append(("policy_documents", n, "uploaded in the app"))
|
||||||
|
|
||||||
|
cur.execute("SELECT COUNT(*) FROM service_documents WHERE storageKey REGEXP %s",
|
||||||
|
(UUID_KEY_SQL,))
|
||||||
|
n = cur.fetchone()[0]
|
||||||
|
if n:
|
||||||
|
findings.append(("service_documents", n, "uploaded in the app (key shape, approximate)"))
|
||||||
|
|
||||||
|
return findings, staged is not None
|
||||||
|
|
||||||
|
|
||||||
|
def report(findings, staged_ok: bool, env: str) -> int:
|
||||||
|
print(f"=== Verificación de datos nativos (env={env}) ===", flush=True)
|
||||||
|
|
||||||
|
if not staged_ok:
|
||||||
|
print(
|
||||||
|
" ! No hay Parquet en migration/output, así que no se pueden verificar\n"
|
||||||
|
" los refs contra el extracto de Access. Ejecute con --stage.",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not findings:
|
||||||
|
print(" Sin filas nativas. Una reimportación completa no destruye nada.", flush=True)
|
||||||
|
return 0 if staged_ok else BLOCKED
|
||||||
|
|
||||||
|
total = sum(n for _, n, _ in findings)
|
||||||
|
print(f" {total} filas existen SÓLO en la plataforma y se perderían:", flush=True)
|
||||||
|
for label, n, detail in findings:
|
||||||
|
print(f" {n:>7} {label:<22} {detail}", flush=True)
|
||||||
|
print(
|
||||||
|
"\n Una reimportación completa vacía estas tablas y las reconstruye desde\n"
|
||||||
|
" Access, que no conoce ninguna de estas filas.\n"
|
||||||
|
"\n Use la sincronización aditiva (run_all.py --sync), que respeta lo\n"
|
||||||
|
" capturado en la plataforma. Para reimportar de todos modos y BORRARLAS,\n"
|
||||||
|
" ejecute run_all.py --force-full.",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
return BLOCKED
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
ap = argparse.ArgumentParser(
|
||||||
|
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||||
|
)
|
||||||
|
ap.add_argument("--env", default="dev")
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
conn = connect(args.env)
|
||||||
|
try:
|
||||||
|
findings, staged_ok = scan(conn)
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
raise SystemExit(report(findings, staged_ok, args.env))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -15,6 +15,12 @@ Then:
|
|||||||
./.venv/bin/python run_all.py --env dev # data only (staging already present)
|
./.venv/bin/python run_all.py --env dev # data only (staging already present)
|
||||||
./.venv/bin/python run_all.py --env prod --stage # re-extract from Access first, then load
|
./.venv/bin/python run_all.py --env prod --stage # re-extract from Access first, then load
|
||||||
|
|
||||||
|
A full pass truncates and rebuilds every table it owns from the Access extract,
|
||||||
|
so anything the platform minted itself — allocated portal NUMids, customers
|
||||||
|
created in the staff UI, OCR-captured policies, app-booked ledger rows, uploaded
|
||||||
|
documents — is destroyed. native_guard.py runs first and refuses when the target
|
||||||
|
database holds any of it; --force-full overrides and deletes them.
|
||||||
|
|
||||||
--sync swaps the truncate+rebuild steps for the additive upsert ones. It reads
|
--sync swaps the truncate+rebuild steps for the additive upsert ones. It reads
|
||||||
the same staged Parquet, so it needs --stage too unless a previous run left
|
the same staged Parquet, so it needs --stage too unless a previous run left
|
||||||
migration/output populated on this machine — which is never true in a
|
migration/output populated on this machine — which is never true in a
|
||||||
@@ -83,6 +89,33 @@ SYNC_STEPS = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# native_guard.py exits with this when the target database holds rows that only
|
||||||
|
# exist in the platform. Kept in step with the constant there.
|
||||||
|
GUARD_BLOCKED = 3
|
||||||
|
|
||||||
|
|
||||||
|
def guard(env: str, force: bool) -> None:
|
||||||
|
"""Stop a full pass that would delete platform-native rows.
|
||||||
|
|
||||||
|
Only the full path needs this: --sync upserts legacy rows against the
|
||||||
|
existing refs and leaves everything else alone, so it cannot lose anything.
|
||||||
|
Run after staging, because the guard verifies legacy refs against the staged
|
||||||
|
Parquet and cannot tell an allocated NUMid from an Access one without it.
|
||||||
|
"""
|
||||||
|
cmd = [PY, str(HERE / "native_guard.py"), "--env", env]
|
||||||
|
print("+ " + " ".join(cmd), flush=True)
|
||||||
|
r = subprocess.run(cmd)
|
||||||
|
if r.returncode == GUARD_BLOCKED and not force:
|
||||||
|
sys.exit(r.returncode)
|
||||||
|
if r.returncode == GUARD_BLOCKED and force:
|
||||||
|
print(
|
||||||
|
"\n! --force-full: continuando y BORRANDO las filas nativas listadas.\n",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
elif r.returncode:
|
||||||
|
sys.exit(r.returncode)
|
||||||
|
|
||||||
|
|
||||||
def run(cmd: list[str], step: int | None = None, total: int | None = None) -> None:
|
def run(cmd: list[str], step: int | None = None, total: int | None = None) -> None:
|
||||||
# The "[paso i/N] name" marker is a contract with the Operaciones screen,
|
# The "[paso i/N] name" marker is a contract with the Operaciones screen,
|
||||||
# which parses the last one to show progress. Emitting it here rather than
|
# which parses the last one to show progress. Emitting it here rather than
|
||||||
@@ -103,6 +136,8 @@ def main() -> None:
|
|||||||
help="re-run the raw staging load first (needs the Access files + mdbtools)")
|
help="re-run the raw staging load first (needs the Access files + mdbtools)")
|
||||||
ap.add_argument("--sync", action="store_true",
|
ap.add_argument("--sync", action="store_true",
|
||||||
help="upsert legacy rows and archive removed legacy rows; preserve manual rows")
|
help="upsert legacy rows and archive removed legacy rows; preserve manual rows")
|
||||||
|
ap.add_argument("--force-full", action="store_true",
|
||||||
|
help="run the full truncate+rebuild even when it deletes platform-native rows")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
steps = SYNC_STEPS if args.sync else STEPS
|
steps = SYNC_STEPS if args.sync else STEPS
|
||||||
@@ -116,6 +151,11 @@ def main() -> None:
|
|||||||
run([PY, str(HERE / "load_staging.py"), "--output-dir", str(HERE / "output")],
|
run([PY, str(HERE / "load_staging.py"), "--output-dir", str(HERE / "output")],
|
||||||
step=1, total=total)
|
step=1, total=total)
|
||||||
|
|
||||||
|
# Deliberately not counted as a step: it is a precondition, it takes a
|
||||||
|
# second, and the Operaciones progress bar parses those numbers.
|
||||||
|
if not args.sync:
|
||||||
|
guard(args.env, args.force_full)
|
||||||
|
|
||||||
for i, step in enumerate(steps, start=1 + offset):
|
for i, step in enumerate(steps, start=1 + offset):
|
||||||
cmd = [PY, str(HERE / step), "--env", args.env]
|
cmd = [PY, str(HERE / step), "--env", args.env]
|
||||||
if args.sync:
|
if args.sync:
|
||||||
|
|||||||
@@ -149,10 +149,11 @@ def main():
|
|||||||
skip_cust = skip_date = skip_dupe = 0
|
skip_cust = skip_date = skip_dupe = 0
|
||||||
|
|
||||||
def add(cid, domain, tdate, amount, currency, *, period=None, reference=None,
|
def add(cid, domain, tdate, amount, currency, *, period=None, reference=None,
|
||||||
typeid=None, check=None, message=None, src_db=None, src_tbl=None, legacy=None):
|
typeid=None, check=None, message=None, src_db=None, src_tbl=None, legacy=None,
|
||||||
|
outstanding=0):
|
||||||
tx.append((str(uuid.uuid4()), cid, domain, typeid, tdate, period, reference,
|
tx.append((str(uuid.uuid4()), cid, domain, typeid, tdate, period, reference,
|
||||||
amount if amount is not None else Decimal(0), currency, None, check,
|
amount if amount is not None else Decimal(0), currency, None, check,
|
||||||
message, 0, src_db, src_tbl, legacy))
|
message, outstanding, src_db, src_tbl, legacy))
|
||||||
|
|
||||||
# Business key of a real cash payment. `folio` is deliberately excluded: it
|
# Business key of a real cash payment. `folio` is deliberately excluded: it
|
||||||
# is a per-table sequential number that collides between EFECTIVO and
|
# is a per-table sequential number that collides between EFECTIVO and
|
||||||
@@ -230,6 +231,16 @@ def main():
|
|||||||
src_db="UTILITIES", src_tbl=legacy_tbl, legacy=str(int(r["_row_num"])))
|
src_db="UTILITIES", src_tbl=legacy_tbl, legacy=str(int(r["_row_num"])))
|
||||||
|
|
||||||
def billing(name, legacy_tbl):
|
def billing(name, legacy_tbl):
|
||||||
|
"""Load a DATOS2-shaped billing ledger.
|
||||||
|
|
||||||
|
NOPAGO is the legacy "still owed" flag. The website reads it directly —
|
||||||
|
`account.statement.php` splits the statement on `NOPAGO = 0` vs
|
||||||
|
`NOPAGO = 1` and renders the latter as the "Outstanding Bills Requiring
|
||||||
|
Attention" table — so dropping it does not merely lose a column, it
|
||||||
|
silently empties that whole section for anyone served off the platform.
|
||||||
|
Only these three tables carry it (76 rows set in DATOS2 today); the
|
||||||
|
EFECTIVO/FM3 cash streams have no such column and stay 0.
|
||||||
|
"""
|
||||||
nonlocal skip_cust, skip_date
|
nonlocal skip_cust, skip_date
|
||||||
df = load("stg_utilities", name)
|
df = load("stg_utilities", name)
|
||||||
for _, r in df.iterrows():
|
for _, r in df.iterrows():
|
||||||
@@ -243,7 +254,8 @@ def main():
|
|||||||
add(cid, "UTILITY", td, dec(r["chargecredit"], Decimal(0)), "MXN",
|
add(cid, "UTILITY", td, dec(r["chargecredit"], Decimal(0)), "MXN",
|
||||||
period=s(r["period"]), reference=s(r["refer"]), typeid=tid,
|
period=s(r["period"]), reference=s(r["refer"]), typeid=tid,
|
||||||
check=s(r["cheque"]), src_db="UTILITIES", src_tbl=legacy_tbl,
|
check=s(r["cheque"]), src_db="UTILITIES", src_tbl=legacy_tbl,
|
||||||
legacy=str(int(r["_row_num"])))
|
legacy=str(int(r["_row_num"])),
|
||||||
|
outstanding=1 if s(r["nopago"]) == "1" else 0)
|
||||||
|
|
||||||
def iva():
|
def iva():
|
||||||
nonlocal skip_cust
|
nonlocal skip_cust
|
||||||
@@ -297,7 +309,7 @@ def main():
|
|||||||
if new_types:
|
if new_types:
|
||||||
c.executemany("INSERT INTO type_transactions (id,nameEn,nameEs,isService) VALUES (%s,%s,%s,%s)", new_types)
|
c.executemany("INSERT INTO type_transactions (id,nameEn,nameEs,isService) VALUES (%s,%s,%s,%s)", new_types)
|
||||||
tx = [(t[0], t[1], t[2], (db_types.get(fresh_name.get(t[3])) if t[3] else None), *t[4:]) for t in tx]
|
tx = [(t[0], t[1], t[2], (db_types.get(fresh_name.get(t[3])) if t[3] else None), *t[4:]) for t in tx]
|
||||||
c.executemany("INSERT INTO transactions (id,customerId,domain,typeId,transactionDate,period,reference,amount,currency,exchangeRate,checkNumber,message,outstanding,legacySourceDb,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),domain=VALUES(domain),typeId=VALUES(typeId),transactionDate=VALUES(transactionDate),period=VALUES(period),reference=VALUES(reference),amount=VALUES(amount),currency=VALUES(currency),checkNumber=VALUES(checkNumber),message=VALUES(message),voidedAt=NULL", tx)
|
c.executemany("INSERT INTO transactions (id,customerId,domain,typeId,transactionDate,period,reference,amount,currency,exchangeRate,checkNumber,message,outstanding,legacySourceDb,legacySourceTable,legacyId) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY UPDATE customerId=VALUES(customerId),domain=VALUES(domain),typeId=VALUES(typeId),transactionDate=VALUES(transactionDate),period=VALUES(period),reference=VALUES(reference),amount=VALUES(amount),currency=VALUES(currency),checkNumber=VALUES(checkNumber),message=VALUES(message),outstanding=VALUES(outstanding),voidedAt=NULL", tx)
|
||||||
else:
|
else:
|
||||||
c.execute("SET FOREIGN_KEY_CHECKS=0")
|
c.execute("SET FOREIGN_KEY_CHECKS=0")
|
||||||
for t in ("transactions", "type_transactions", "exchange_rates"):
|
for t in ("transactions", "type_transactions", "exchange_rates"):
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "jorgecuadros-platform",
|
"name": "jorgecuadros-platform",
|
||||||
"version": "1.0.14",
|
"version": "1.0.17",
|
||||||
"private": true,
|
"private": true,
|
||||||
"workspaces": [
|
"workspaces": [
|
||||||
"apps/*",
|
"apps/*",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@jorgecuadros/database",
|
"name": "@jorgecuadros/database",
|
||||||
"version": "1.0.14",
|
"version": "1.0.17",
|
||||||
"private": true,
|
"private": true,
|
||||||
"main": "generated/client/index.js",
|
"main": "generated/client/index.js",
|
||||||
"types": "generated/client/index.d.ts",
|
"types": "generated/client/index.d.ts",
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* NUMid recycle audit.
|
||||||
|
*
|
||||||
|
* Prints which portal NUMids (customer_legacy_refs, utilities/DATGRAL) are dead
|
||||||
|
* enough to hand to a new customer, and which are merely quiet. Read-only: it
|
||||||
|
* writes nothing and reassigns nothing.
|
||||||
|
*
|
||||||
|
* node scripts/numid-audit.mjs # summary + both candidate tiers
|
||||||
|
* node scripts/numid-audit.mjs --csv # full per-NUMid table on stdout
|
||||||
|
* node scripts/numid-audit.mjs --numid 501
|
||||||
|
*
|
||||||
|
* Needs DATABASE_URL. Run it against PROD before acting on anything: the tiers
|
||||||
|
* describe whatever database it is pointed at, and a stale copy will happily
|
||||||
|
* report a NUMid as empty that prod has been billing all year.
|
||||||
|
*/
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { dirname, join } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
// Relative, not "@jorgecuadros/database": the workspace link is not always
|
||||||
|
// present at the repo root, and this script has to run from a bare checkout and
|
||||||
|
// from inside the API container alike.
|
||||||
|
import pkg from "../packages/database/generated/client/index.js";
|
||||||
|
|
||||||
|
const { PrismaClient } = pkg;
|
||||||
|
|
||||||
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Anything here means the id carries history a new owner would inherit.
|
||||||
|
* Counted, not sampled: a single row in any of them disqualifies.
|
||||||
|
*/
|
||||||
|
const HISTORY_COLUMNS = [
|
||||||
|
"veh",
|
||||||
|
"trust",
|
||||||
|
"stmt",
|
||||||
|
"ocr",
|
||||||
|
"enl",
|
||||||
|
"elog",
|
||||||
|
"ash",
|
||||||
|
"nopago",
|
||||||
|
];
|
||||||
|
|
||||||
|
const n = (v) => (v == null ? 0 : Number(v));
|
||||||
|
const hasHistory = (r) => HISTORY_COLUMNS.some((c) => n(r[c]) > 0);
|
||||||
|
const zeroBalance = (r) => n(r.balMxn) === 0 && n(r.balUsd) === 0;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* EMPTY — the id was created and never used. Safe for an allocator to take
|
||||||
|
* without a human looking, subject to the legacy check below.
|
||||||
|
*
|
||||||
|
* "Never transacted" means zero movements once the synthetic opening-balance row
|
||||||
|
* is removed; that row exists for all 1,171 NUMids and is not evidence of use.
|
||||||
|
* Services are checked as anySvc rather than activeSvc, because a deactivated
|
||||||
|
* water account still says a person once lived behind this id.
|
||||||
|
*/
|
||||||
|
const isEmpty = (r) =>
|
||||||
|
n(r.realTx) === 0 &&
|
||||||
|
zeroBalance(r) &&
|
||||||
|
n(r.anySvc) === 0 &&
|
||||||
|
n(r.anyPol) === 0 &&
|
||||||
|
n(r.insRef) === 0 &&
|
||||||
|
n(r.hasEmail) === 0 &&
|
||||||
|
!hasHistory(r);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DORMANT — used once, quiet for years, owes nothing. NOT auto-allocatable:
|
||||||
|
* a returning snowbird is indistinguishable from an abandoned account here.
|
||||||
|
*/
|
||||||
|
const isDormant = (r) =>
|
||||||
|
!isEmpty(r) &&
|
||||||
|
n(r.realTx36m) === 0 &&
|
||||||
|
zeroBalance(r) &&
|
||||||
|
n(r.activeSvc) === 0 &&
|
||||||
|
n(r.activePol) === 0 &&
|
||||||
|
!hasHistory(r);
|
||||||
|
|
||||||
|
function line(r) {
|
||||||
|
return (
|
||||||
|
` ${String(r.numid).padStart(5)} ${(r.name || "(sin nombre)").slice(0, 30).padEnd(30)}` +
|
||||||
|
` last=${(r.lastRealTx ? new Date(r.lastRealTx).toISOString().slice(0, 10) : "never").padStart(10)}` +
|
||||||
|
` tx=${String(n(r.realTx)).padStart(3)}` +
|
||||||
|
` bal=${n(r.balMxn).toFixed(2).padStart(10)}` +
|
||||||
|
` svc=${n(r.anySvc)}` +
|
||||||
|
` pol=${n(r.anyPol)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const args = process.argv.slice(2);
|
||||||
|
const prisma = new PrismaClient();
|
||||||
|
try {
|
||||||
|
const sql = readFileSync(join(HERE, "numid-audit.sql"), "utf8");
|
||||||
|
const rows = await prisma.$queryRawUnsafe(sql);
|
||||||
|
|
||||||
|
const one = args.indexOf("--numid");
|
||||||
|
if (one !== -1) {
|
||||||
|
const want = Number(args[one + 1]);
|
||||||
|
const r = rows.find((x) => Number(x.numid) === want);
|
||||||
|
if (!r) {
|
||||||
|
console.log(`NUMid ${want} is not in the utilities/DATGRAL pool.`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
console.log(JSON.stringify(r, (_k, v) => (typeof v === "bigint" ? Number(v) : v), 2));
|
||||||
|
console.log(
|
||||||
|
`\nverdict: ${isEmpty(r) ? "EMPTY" : isDormant(r) ? "DORMANT" : "IN USE"}`,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.includes("--csv")) {
|
||||||
|
const cols = Object.keys(rows[0]);
|
||||||
|
console.log(cols.join(","));
|
||||||
|
for (const r of rows) {
|
||||||
|
console.log(cols.map((c) => JSON.stringify(r[c] ?? "")).join(","));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const empty = rows.filter(isEmpty);
|
||||||
|
const dormant = rows.filter(isDormant);
|
||||||
|
const max = rows.reduce((m, r) => Math.max(m, Number(r.numid)), 0);
|
||||||
|
|
||||||
|
console.log(`pool: ${rows.length} NUMids, max ${max}`);
|
||||||
|
console.log(` EMPTY (never used, auto-allocatable): ${empty.length}`);
|
||||||
|
console.log(` DORMANT (quiet, needs a human): ${dormant.length}`);
|
||||||
|
console.log(` IN USE: ${rows.length - empty.length - dormant.length}`);
|
||||||
|
|
||||||
|
console.log("\nEMPTY");
|
||||||
|
empty.forEach((r) => console.log(line(r)));
|
||||||
|
console.log("\nDORMANT");
|
||||||
|
dormant.forEach((r) => console.log(line(r)));
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
"\nNOTE: this audit sees the platform only. Every NUMid here also exists in\n" +
|
||||||
|
"Access, and freakma republishes DATGRAL in full on each export, so an id\n" +
|
||||||
|
"reassigned here comes back under its old owner unless it is removed at the\n" +
|
||||||
|
"source or the NUMid is routed to the platform. Confirm against prod\n" +
|
||||||
|
"datosfreak before reassigning.",
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await prisma.$disconnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
-- One row per portal NUMid, with every signal that says whether the id is in use.
|
||||||
|
-- Consumed by scripts/numid-audit.mjs, which applies the tier rules.
|
||||||
|
--
|
||||||
|
-- POOL. customer_legacy_refs where sourceSystem='utilities' AND sourceTable='DATGRAL'.
|
||||||
|
-- That pair IS the portal "Security Number" the login screen asks for.
|
||||||
|
-- insurance/DATGRAL is a DIFFERENT id space running to 4000 and sharing the same
|
||||||
|
-- sourceTable name; drawing from it would hand out an id the portal cannot resolve.
|
||||||
|
--
|
||||||
|
-- WHY THE OBVIOUS RULES FIND NOTHING.
|
||||||
|
-- "every owned row count is zero" -> 0 of 1,171. Migration gave every NUMid
|
||||||
|
-- at least one property and one transaction.
|
||||||
|
-- "no transaction in the last N years" -> 0 of 1,171. Every customer carries a
|
||||||
|
-- synthetic Jan-1 opening-balance row, so
|
||||||
|
-- everyone looks active in the current year.
|
||||||
|
-- The opening-balance row has to be subtracted before any of this means anything,
|
||||||
|
-- which is what `bf` below does and why `real_tx` exists.
|
||||||
|
--
|
||||||
|
-- BALANCE-FORWARD DETECTION IS TWO-SHAPED ON PURPOSE.
|
||||||
|
-- transform_transactions.py:120 mints a transaction type literally named
|
||||||
|
-- 'BALANCE FORWARD'. Databases loaded before that change carry the same rows with
|
||||||
|
-- typeId NULL, dated Jan 1, legacySourceTable='datos2' -- 1,170 of them, exactly one
|
||||||
|
-- per customer. Matching the type name alone floors nothing on such a database, and
|
||||||
|
-- every balance below silently becomes a raw lifetime sum: the same double-count that
|
||||||
|
-- read the whole book as +20.6M MXN in credit before d173c9e. Match both shapes.
|
||||||
|
--
|
||||||
|
-- Balances otherwise follow BillingService exactly -- voided out, outstanding out,
|
||||||
|
-- superseded rows out (BALANCE_FLOOR_JOIN / NOT_SUPERSEDED, billing.service.ts:179-210).
|
||||||
|
|
||||||
|
WITH bf AS (
|
||||||
|
SELECT t.id, t.customerId, t.transactionDate
|
||||||
|
FROM transactions t
|
||||||
|
LEFT JOIN type_transactions tt ON tt.id = t.typeId
|
||||||
|
WHERE t.voidedAt IS NULL
|
||||||
|
AND (
|
||||||
|
tt.nameEn = 'BALANCE FORWARD'
|
||||||
|
OR (t.typeId IS NULL AND MONTH(t.transactionDate) = 1 AND DAY(t.transactionDate) = 1
|
||||||
|
AND t.legacySourceTable = 'datos2')
|
||||||
|
)
|
||||||
|
),
|
||||||
|
bfloor AS (
|
||||||
|
SELECT customerId, MAX(transactionDate) AS floorDate FROM bf GROUP BY customerId
|
||||||
|
),
|
||||||
|
real_tx AS (
|
||||||
|
SELECT t.* FROM transactions t
|
||||||
|
WHERE t.voidedAt IS NULL AND t.id NOT IN (SELECT id FROM bf)
|
||||||
|
),
|
||||||
|
pool AS (
|
||||||
|
SELECT CAST(r.legacyId AS UNSIGNED) AS numid,
|
||||||
|
c.id AS cid,
|
||||||
|
REPLACE(REPLACE(COALESCE(c.name,''),'\n',' '),'\t',' ') AS name,
|
||||||
|
IF(c.archivedAt IS NULL,0,1) AS archived,
|
||||||
|
IF(c.email IS NULL OR c.email='',0,1) AS hasEmail
|
||||||
|
FROM customer_legacy_refs r
|
||||||
|
JOIN customers c ON c.id = r.customerId
|
||||||
|
WHERE r.sourceSystem='utilities' AND r.sourceTable='DATGRAL'
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
p.numid, p.cid AS customerUuid, p.name, p.archived, p.hasEmail,
|
||||||
|
-- EXISTS, not a join: 16 customers hold more than one insurance ref (several
|
||||||
|
-- insurance rows folded into one customer), and joining them fans this result
|
||||||
|
-- out past one row per NUMid — 1,188 rows for a 1,171-id pool.
|
||||||
|
EXISTS(SELECT 1 FROM customer_legacy_refs i
|
||||||
|
WHERE i.customerId=p.cid AND i.sourceSystem='insurance') AS insRef,
|
||||||
|
|
||||||
|
COALESCE((SELECT ROUND(SUM(t.amount),2) FROM transactions t
|
||||||
|
LEFT JOIN bfloor f ON f.customerId=t.customerId
|
||||||
|
WHERE t.customerId=p.cid AND t.voidedAt IS NULL AND t.outstanding=0
|
||||||
|
AND t.currency='MXN'
|
||||||
|
AND (f.floorDate IS NULL OR t.transactionDate>=f.floorDate)),0) AS balMxn,
|
||||||
|
COALESCE((SELECT ROUND(SUM(t.amount),2) FROM transactions t
|
||||||
|
LEFT JOIN bfloor f ON f.customerId=t.customerId
|
||||||
|
WHERE t.customerId=p.cid AND t.voidedAt IS NULL AND t.outstanding=0
|
||||||
|
AND t.currency='USD'
|
||||||
|
AND (f.floorDate IS NULL OR t.transactionDate>=f.floorDate)),0) AS balUsd,
|
||||||
|
|
||||||
|
(SELECT COUNT(*) FROM transactions t
|
||||||
|
WHERE t.customerId=p.cid AND t.voidedAt IS NULL AND t.outstanding=1) AS nopago,
|
||||||
|
(SELECT COUNT(*) FROM real_tx t WHERE t.customerId=p.cid) AS realTx,
|
||||||
|
(SELECT COUNT(*) FROM real_tx t WHERE t.customerId=p.cid
|
||||||
|
AND t.transactionDate >= DATE_SUB(CURDATE(), INTERVAL 12 MONTH)) AS realTx12m,
|
||||||
|
(SELECT COUNT(*) FROM real_tx t WHERE t.customerId=p.cid
|
||||||
|
AND t.transactionDate >= DATE_SUB(CURDATE(), INTERVAL 36 MONTH)) AS realTx36m,
|
||||||
|
(SELECT DATE(MAX(t.transactionDate)) FROM real_tx t WHERE t.customerId=p.cid) AS lastRealTx,
|
||||||
|
|
||||||
|
(SELECT COUNT(*) FROM properties pr WHERE pr.customerId=p.cid AND pr.archivedAt IS NULL) AS props,
|
||||||
|
-- services are counted BOTH ways: an inactive service is still a record of the id
|
||||||
|
-- having been used, so the auto tier requires zero of any kind.
|
||||||
|
(SELECT COUNT(*) FROM property_services ps JOIN properties pr ON pr.id=ps.propertyId
|
||||||
|
WHERE pr.customerId=p.cid AND pr.archivedAt IS NULL) AS anySvc,
|
||||||
|
(SELECT COUNT(*) FROM property_services ps JOIN properties pr ON pr.id=ps.propertyId
|
||||||
|
WHERE pr.customerId=p.cid AND pr.archivedAt IS NULL AND ps.active=1) AS activeSvc,
|
||||||
|
(SELECT COUNT(*) FROM policies po WHERE po.customerId=p.cid AND po.archivedAt IS NULL
|
||||||
|
AND (po.policyTo IS NULL OR po.policyTo >= CURDATE())) AS activePol,
|
||||||
|
(SELECT COUNT(*) FROM policies po WHERE po.customerId=p.cid AND po.archivedAt IS NULL) AS anyPol,
|
||||||
|
(SELECT COUNT(*) FROM vehicles v WHERE v.customerId=p.cid) AS veh,
|
||||||
|
(SELECT COUNT(*) FROM trust_accounts ta JOIN properties pr ON pr.id=ta.propertyId
|
||||||
|
WHERE pr.customerId=p.cid) AS trust,
|
||||||
|
(SELECT COUNT(*) FROM statement_documents s WHERE s.matchedCustomerId=p.cid) AS stmt,
|
||||||
|
(SELECT COUNT(*) FROM policy_ocr_documents o WHERE o.matchedCustomerId=p.cid) AS ocr,
|
||||||
|
(SELECT COUNT(*) FROM email_notification_log e WHERE e.customerId=p.cid) AS enl,
|
||||||
|
(SELECT COUNT(*) FROM email_log e WHERE e.customerId=p.cid) AS elog,
|
||||||
|
(SELECT COUNT(*) FROM account_status_history a WHERE a.customerId=p.cid) AS ash
|
||||||
|
FROM pool p
|
||||||
|
ORDER BY p.numid;
|
||||||
Reference in New Issue
Block a user