rmancinasandClaude Opus 5
898cf48c80
Every transform resolves its target through dbenv.database_url(), which lets a
DATABASE_URL in the process environment win — that is how the API container
drives a re-import against its own database with no deploy/ directory present.
blob_extract.py was the one step that bypassed it and called load_env()
directly for the MinIO credentials, so the "Operaciones" re-import loaded all
the data and then exited 1 on:
missing /repo/deploy/.env.prod — deploy the 'prod' DB stack and write its
.env first
Give the S3 settings the same resolution as the DB URL: load_env() now returns
{} for an absent file, and setting()/require() layer the process environment on
top of it. blob_extract reads S3_ENDPOINT / S3_BUCKET and accepts either
S3_ACCESS_KEY/S3_SECRET_KEY or MINIO_ROOT_USER/MINIO_ROOT_PASSWORD, matching
the fallback order in storage.service.ts and the vars the api service already
sets in deploy/galactus/jorgecuadros-app.compose.yml. A genuinely missing
setting still fails fast, now naming the variable.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
106 lines
3.6 KiB
Python
106 lines
3.6 KiB
Python
"""
|
|
Environment selection for the migration scripts.
|
|
|
|
Every transform is environment-agnostic: it reads the staged Parquet (same for
|
|
all environments) and writes to whichever database `--env` selects. The target
|
|
is defined entirely by `deploy/.env.<env>` (the same file Portainer is fed at
|
|
deploy time), whose `DATABASE_URL` is the single source of truth for host /
|
|
port / credentials / database.
|
|
|
|
Reproduce the whole migration in a new environment (e.g. prod) by:
|
|
1. deploy the DB stack for that env (deploy/jorgecuadros-db.stack.yml)
|
|
2. write deploy/.env.<env> with its DATABASE_URL
|
|
3. push the schema: DATABASE_URL=... npx prisma@5 db push --schema=packages/database/prisma/schema.prisma
|
|
4. run: ./.venv/bin/python run_all.py --env <env>
|
|
|
|
Usage in a script:
|
|
from dbenv import connect, env_arg
|
|
env = env_arg() # --env dev|prod (default dev)
|
|
conn = connect(env)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
from pathlib import Path
|
|
from urllib.parse import unquote, urlparse
|
|
|
|
import pymysql
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def load_env(env: str) -> dict:
|
|
"""deploy/.env.<env> parsed to a dict, or {} when the file is absent.
|
|
|
|
Absent is normal, not an error: the API container runs these scripts with
|
|
DATABASE_URL / S3_* injected as real environment variables and ships no
|
|
deploy/ directory. Use `setting()` / `require()` rather than this — they
|
|
layer the process environment on top, which is what actually resolves."""
|
|
f = REPO / "deploy" / f".env.{env}"
|
|
if not f.exists():
|
|
return {}
|
|
out = {}
|
|
for line in f.read_text().splitlines():
|
|
line = line.strip()
|
|
if line and not line.startswith("#") and "=" in line:
|
|
k, v = line.split("=", 1)
|
|
out[k] = v
|
|
return out
|
|
|
|
|
|
def setting(env: str, *keys: str):
|
|
"""First non-empty value for `keys`, process environment first, then
|
|
deploy/.env.<env>. Several keys = fallback aliases (S3_ACCESS_KEY then
|
|
MINIO_ROOT_USER, as apps/api/src/storage/storage.service.ts does)."""
|
|
fromfile = load_env(env)
|
|
for k in keys:
|
|
v = os.environ.get(k) or fromfile.get(k)
|
|
if v:
|
|
return v
|
|
return None
|
|
|
|
|
|
def require(env: str, *keys: str) -> str:
|
|
v = setting(env, *keys)
|
|
if not v:
|
|
raise SystemExit(
|
|
f"missing {' / '.join(keys)} — set it in the environment, or deploy the "
|
|
f"'{env}' stack and write {REPO / 'deploy' / f'.env.{env}'} "
|
|
f"(see dbenv.py header)."
|
|
)
|
|
return v
|
|
|
|
|
|
def database_url(env: str) -> str:
|
|
"""Target DB URL. A DATABASE_URL in the process environment wins over
|
|
deploy/.env.<env> — this is how the API container (which has its own
|
|
DATABASE_URL and no deploy/.env files) drives a re-import against its own
|
|
database."""
|
|
return require(env, "DATABASE_URL")
|
|
|
|
|
|
def connect(env: str):
|
|
url = database_url(env)
|
|
u = urlparse(url) # mysql://user:pass@host:port/db
|
|
return pymysql.connect(
|
|
host=u.hostname,
|
|
port=u.port or 3306,
|
|
user=unquote(u.username or ""),
|
|
password=unquote(u.password or ""),
|
|
database=(u.path or "/").lstrip("/"),
|
|
autocommit=False,
|
|
charset="utf8mb4",
|
|
)
|
|
|
|
|
|
def env_arg(extra_args=None) -> str:
|
|
"""Parse --env (default 'dev') and return it. Scripts that need more args
|
|
can pass an argparse parser via extra_args(parser)."""
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--env", default="dev", help="target environment (dev|prod|...): reads deploy/.env.<env>")
|
|
if extra_args:
|
|
extra_args(p)
|
|
return p.parse_args().env
|