Move strr-api deployed DB access to Cloud SQL IAM auth

strr-api/.env.sample

@@ -33,6 +33,7 @@ DATABASE_PORT=15432
 # Optional: Alembic runs migrations with this DB role so new objects are owned by it.
 # For Cloud SQL IAM auth, set this to the service account's Postgres role name.
 DATABASE_OWNER_ROLE=
+DATABASE_MIGRATION_USERNAME=
 
 ## TEST DB
 DATABASE_TEST_USERNAME=postgres

strr-api/devops/vaults.gcp.env

@@ -14,10 +14,11 @@ KEYCLOAK_AUTH_TOKEN_URL="op://keycloak/$APP_ENV/base/KEYCLOAK_AUTH_TOKEN_URL"
 STRR_SERVICE_ACCOUNT_CLIENT_ID="op://keycloak/$APP_ENV/strr-service-account/STRR_SERVICE_ACCOUNT_CLIENT_ID"
 STRR_SERVICE_ACCOUNT_SECRET="op://keycloak/$APP_ENV/strr-service-account/STRR_SERVICE_ACCOUNT_SECRET"
 DATABASE_NAME="op://database/$APP_ENV/strr-db/DATABASE_NAME"
-DATABASE_PASSWORD="op://database/$APP_ENV/strr-db/DATABASE_PASSWORD"
-DATABASE_PORT="op://database/$APP_ENV/strr-db/DATABASE_PORT"
-DATABASE_UNIX_SOCKET="op://database/$APP_ENV/strr-db/DATABASE_UNIX_SOCKET"
-DATABASE_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_USERNAME"
+DATABASE_HOST="127.0.0.1"
+DATABASE_PORT="5432"
+DATABASE_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_IAM_USERNAME"
+DATABASE_MIGRATION_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_MIGRATION_IAM_USERNAME"
+DATABASE_OWNER_ROLE="op://database/$APP_ENV/strr-db/DATABASE_OWNER_ROLE"
 JWT_OIDC_AUDIENCE="op://keycloak/$APP_ENV/account-services-account/ACCOUNT_SERVICES_SERVICE_ACCOUNT_CLIENT_ID"
 JWT_OIDC_JWKS_CACHE_TIMEOUT="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_JWKS_CACHE_TIMEOUT"
 JWT_OIDC_WELL_KNOWN_CONFIG="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_WELL_KNOWN_CONFIG"

strr-api/src/strr_api/config.py

@@ -45,11 +45,79 @@
 import os
 
 from dotenv import find_dotenv, load_dotenv
+from sqlalchemy.engine import URL
 
 basedir = os.path.abspath(os.path.dirname(__file__))
 
 load_dotenv(find_dotenv())
 
+GCP_DEPLOYMENT_ENVS = {"development", "test", "uat", "sandbox", "production", "migration"}
+PROXY_REQUIRED_ENVS = ("DATABASE_NAME",)
+
+
+def _deployment_env() -> str:
+    return os.getenv("DEPLOYMENT_ENV", os.getenv("POD_NAMESPACE", "local"))
+
+
+def _is_deployed_gcp() -> bool:
+    return bool(os.getenv("K_SERVICE")) or _deployment_env() in GCP_DEPLOYMENT_ENVS
+
+
+def _use_proxy_iam() -> bool:
+    return _is_deployed_gcp()
+
+
+def _cloudsql_user_env() -> str:
+    return "DATABASE_MIGRATION_USERNAME" if _deployment_env() == "migration" else "DATABASE_USERNAME"
+
+
+def _require_proxy_env(user_env: str):
+    required = (*PROXY_REQUIRED_ENVS, user_env)
+    missing = [env_name for env_name in required if not os.getenv(env_name)]
+    if missing:
+        raise RuntimeError(f"Missing Cloud SQL IAM proxy environment variables: {', '.join(missing)}")
+
+
+def _proxy_database_uri(user_env: str) -> str:
+    db_user = os.environ[user_env]
+    db_name = os.environ["DATABASE_NAME"]
+    db_host = os.getenv("DATABASE_HOST", "127.0.0.1")
+    db_port = int(os.getenv("DATABASE_PORT", "5432"))
+
+    if db_unix_socket := os.getenv("DATABASE_UNIX_SOCKET", None):
+        return str(
+            URL.create(
+                "postgresql+psycopg2",
+                username=db_user,
+                database=db_name,
+                query={"host": db_unix_socket},
+            )
+        )
+
+    return str(URL.create("postgresql+psycopg2", username=db_user, host=db_host, port=db_port, database=db_name))
+
+
+def _local_database_uri() -> str:
+    db_user = os.getenv("DATABASE_USERNAME", "")
+    db_password = os.getenv("DATABASE_PASSWORD", "")
+    db_name = os.getenv("DATABASE_NAME", "")
+    db_host = os.getenv("DATABASE_HOST", "")
+    db_port = int(os.getenv("DATABASE_PORT", "5432"))
+
+    if db_unix_socket := os.getenv("DATABASE_UNIX_SOCKET", None):
+        return f"postgresql+psycopg2://{db_user}:{db_password}@/{db_name}?host={db_unix_socket}"
+
+    return f"postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db_name}"
+
+
+def _database_settings() -> tuple[str, dict]:
+    if _use_proxy_iam():
+        user_env = _cloudsql_user_env()
+        _require_proxy_env(user_env)
+        return _proxy_database_uri(user_env), {}
+
+    return _local_database_uri(), {}
+
 
 class Config:  # pylint: disable=too-few-public-methods
     """Base class configuration that should set reasonable defaults.
@@ -62,20 +130,10 @@ class Config:  # pylint: disable=too-few-public-methods
     CSRF_ENABLED = True
     SECRET_KEY = "this-really-needs-to-be-changed"
     PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__))
-    POD_NAMESPACE = os.getenv("DEPLOYMENT_ENV", "production")
+    POD_NAMESPACE = _deployment_env()
 
     SQLALCHEMY_TRACK_MODIFICATIONS = False
-
-    DB_USER = os.getenv("DATABASE_USERNAME", "")
-    DB_PASSWORD = os.getenv("DATABASE_PASSWORD", "")
-    DB_NAME = os.getenv("DATABASE_NAME", "")
-    DB_HOST = os.getenv("DATABASE_HOST", "")
-    DB_PORT = int(os.getenv("DATABASE_PORT", "5432"))  # POSTGRESQL
-    # POSTGRESQL
-    if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None):
-        SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg2://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}"
-    else:
-        SQLALCHEMY_DATABASE_URI = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
+    SQLALCHEMY_DATABASE_URI, SQLALCHEMY_ENGINE_OPTIONS = _database_settings()
 
     LD_SDK_KEY = os.getenv("LD_SDK_KEY", None)
 
@@ -158,19 +216,7 @@ class Migration(Config):  # pylint: disable=too-few-public-methods
 
     TESTING = False
     DEBUG = True
-
-    # POSTGRESQL
-    DB_USER = os.getenv("DATABASE_USERNAME", "")
-    DB_PASSWORD = os.getenv("DATABASE_PASSWORD", "")
-    DB_NAME = os.getenv("DATABASE_NAME", "")
-    DB_HOST = os.getenv("DATABASE_HOST", "")
-    DB_PORT = int(os.getenv("DATABASE_PORT", "5432"))  # POSTGRESQL
-    if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None):
-        SQLALCHEMY_DATABASE_URI = (
-            f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?unix_sock={DB_UNIX_SOCKET}/.s.PGSQL.5432"
-        )
-    else:
-        SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
+    SQLALCHEMY_DATABASE_URI, SQLALCHEMY_ENGINE_OPTIONS = _database_settings()
 
 
 class Testing(Config):  # pylint: disable=too-few-public-methods
@@ -188,6 +234,7 @@ class Testing(Config):  # pylint: disable=too-few-public-methods
         f"postgresql://{DATABASE_TEST_USERNAME}:{DATABASE_TEST_PASSWORD}@"
         f"{DATABASE_TEST_HOST}:{DATABASE_TEST_PORT}/{DATABASE_TEST_NAME}"
     )
+    SQLALCHEMY_ENGINE_OPTIONS = {}
 
     AUTH_SVC_URL = "https://test-auth-svc-url"
     PAYMENT_SVC_URL = "https://test-pay-url"

strr-api/tests/integration/test_alembic_ownership.py

@@ -2,17 +2,33 @@
 
 from pathlib import Path
 
+import docker
+import pytest
 from alembic import command
 from alembic.config import Config
+from docker.errors import DockerException
 from sqlalchemy import create_engine, text
 from testcontainers.postgres import PostgresContainer
 
 
+def _require_docker():
+    try:
+        client = docker.from_env()
+        client.ping()
+    except DockerException as exc:
+        pytest.skip(f"Docker is required for this integration test: {exc}")
+    finally:
+        if "client" in locals():
+            client.close()
+
+
 def test_alembic_runs_with_configured_owner_role(monkeypatch):
     """Alembic runs migrations with the configured DB owner role."""
+    _require_docker()
+
     api_root = Path(__file__).resolve().parents[2]
     migrations_path = api_root / "migrations"
-    owner = "sa-api@bcrbk9-test.iam"
+    owner = "strr"
 
     with PostgresContainer("postgres:16-alpine") as postgres:
         db_url = postgres.get_connection_url()

strr-api/tests/unit/test_config_cloudsql_iam.py

@@ -0,0 +1,106 @@
+"""Tests for strr-api Cloud SQL IAM database configuration."""
+
+import importlib
+
+import dotenv
+import pytest
+
+ENV_KEYS = (
+    "DATABASE_HOST",
+    "DATABASE_MIGRATION_USERNAME",
+    "DATABASE_NAME",
+    "DATABASE_PASSWORD",
+    "DATABASE_PORT",
+    "DATABASE_UNIX_SOCKET",
+    "DATABASE_USERNAME",
+    "DEPLOYMENT_ENV",
+    "K_SERVICE",
+    "POD_NAMESPACE",
+)
+
+
+def _clear_env(monkeypatch):
+    for env_name in ENV_KEYS:
+        monkeypatch.delenv(env_name, raising=False)
+
+
+def _reload_config(monkeypatch, **env):
+    _clear_env(monkeypatch)
+    # Keep these tests independent from each developer's local .env file.
+    monkeypatch.setattr(dotenv, "load_dotenv", lambda *_args, **_kwargs: False)
+    for env_name, value in env.items():
+        monkeypatch.setenv(env_name, value)
+
+    from strr_api import config as config_module
+
+    return importlib.reload(config_module)
+
+
+def test_production_config_uses_cloudsql_proxy_iam_uri(monkeypatch):
+    config_module = _reload_config(
+        monkeypatch,
+        DEPLOYMENT_ENV="production",
+        DATABASE_HOST="127.0.0.1",
+        DATABASE_NAME="strr-db",
+        DATABASE_PORT="5432",
+        DATABASE_USERNAME="sa-api@bcrbk9-prod.iam",
+    )
+
+    assert (
+        config_module.Production.SQLALCHEMY_DATABASE_URI
+        == "postgresql+psycopg2://sa-api%40bcrbk9-prod.iam@127.0.0.1:5432/strr-db"
+    )
+    assert config_module.Production.SQLALCHEMY_ENGINE_OPTIONS == {}
+
+
+def test_migration_mode_uses_migration_iam_username(monkeypatch):
+    config_module = _reload_config(
+        monkeypatch,
+        DEPLOYMENT_ENV="migration",
+        DATABASE_HOST="127.0.0.1",
+        DATABASE_MIGRATION_USERNAME="sa-db-migrate@bcrbk9-dev.iam",
+        DATABASE_NAME="strr-db",
+        DATABASE_PORT="5432",
+        DATABASE_USERNAME="sa-api@bcrbk9-dev.iam",
+    )
+
+    assert (
+        config_module.Migration.SQLALCHEMY_DATABASE_URI
+        == "postgresql+psycopg2://sa-db-migrate%40bcrbk9-dev.iam@127.0.0.1:5432/strr-db"
+    )
+
+
+def test_production_config_can_use_cloudsql_proxy_unix_socket(monkeypatch):
+    config_module = _reload_config(
+        monkeypatch,
+        DEPLOYMENT_ENV="production",
+        DATABASE_NAME="strr-db",
+        DATABASE_UNIX_SOCKET="/cloudsql/bcrbk9-dev:northamerica-northeast1:strr-db-dev",
+        DATABASE_USERNAME="sa-api@bcrbk9-dev.iam",
+    )
+
+    assert (
+        config_module.Production.SQLALCHEMY_DATABASE_URI == "postgresql+psycopg2://sa-api%40bcrbk9-dev.iam@/strr-db"
+        "?host=%2Fcloudsql%2Fbcrbk9-dev%3Anorthamerica-northeast1%3Astrr-db-dev"
+    )
+    assert config_module.Production.SQLALCHEMY_ENGINE_OPTIONS == {}
+
+
+def test_deployed_config_requires_cloudsql_proxy_iam_env(monkeypatch):
+    with pytest.raises(RuntimeError, match="DATABASE_NAME, DATABASE_USERNAME"):
+        _reload_config(monkeypatch, DEPLOYMENT_ENV="production")
+
+
+def test_local_config_keeps_password_database_uri(monkeypatch):
+    config_module = _reload_config(
+        monkeypatch,
+        DATABASE_HOST="localhost",
+        DATABASE_NAME="postgres",
+        DATABASE_PASSWORD="postgres",
+        DATABASE_PORT="15432",
+        DATABASE_USERNAME="postgres",
+    )
+
+    assert config_module.Production.POD_NAMESPACE == "local"
+    assert config_module.Production.SQLALCHEMY_DATABASE_URI == "postgresql://postgres:postgres@localhost:15432/postgres"
+    assert config_module.Production.SQLALCHEMY_ENGINE_OPTIONS == {}