diff --git a/strr-api/.env.sample b/strr-api/.env.sample index a3e5e49df..91f19b60f 100644 --- a/strr-api/.env.sample +++ b/strr-api/.env.sample @@ -33,6 +33,7 @@ DATABASE_PORT=15432 # Optional: Alembic runs migrations with this DB role so new objects are owned by it. # For Cloud SQL IAM auth, set this to the service account's Postgres role name. DATABASE_OWNER_ROLE= +DATABASE_MIGRATION_USERNAME= ## TEST DB DATABASE_TEST_USERNAME=postgres diff --git a/strr-api/devops/vaults.gcp.env b/strr-api/devops/vaults.gcp.env index 3d619cfd3..4a4ccd06d 100644 --- a/strr-api/devops/vaults.gcp.env +++ b/strr-api/devops/vaults.gcp.env @@ -14,10 +14,11 @@ KEYCLOAK_AUTH_TOKEN_URL="op://keycloak/$APP_ENV/base/KEYCLOAK_AUTH_TOKEN_URL" STRR_SERVICE_ACCOUNT_CLIENT_ID="op://keycloak/$APP_ENV/strr-service-account/STRR_SERVICE_ACCOUNT_CLIENT_ID" STRR_SERVICE_ACCOUNT_SECRET="op://keycloak/$APP_ENV/strr-service-account/STRR_SERVICE_ACCOUNT_SECRET" DATABASE_NAME="op://database/$APP_ENV/strr-db/DATABASE_NAME" -DATABASE_PASSWORD="op://database/$APP_ENV/strr-db/DATABASE_PASSWORD" -DATABASE_PORT="op://database/$APP_ENV/strr-db/DATABASE_PORT" -DATABASE_UNIX_SOCKET="op://database/$APP_ENV/strr-db/DATABASE_UNIX_SOCKET" -DATABASE_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_USERNAME" +DATABASE_HOST="127.0.0.1" +DATABASE_PORT="5432" +DATABASE_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_IAM_USERNAME" +DATABASE_MIGRATION_USERNAME="op://database/$APP_ENV/strr-db/DATABASE_MIGRATION_IAM_USERNAME" +DATABASE_OWNER_ROLE="op://database/$APP_ENV/strr-db/DATABASE_OWNER_ROLE" JWT_OIDC_AUDIENCE="op://keycloak/$APP_ENV/account-services-account/ACCOUNT_SERVICES_SERVICE_ACCOUNT_CLIENT_ID" JWT_OIDC_JWKS_CACHE_TIMEOUT="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_JWKS_CACHE_TIMEOUT" JWT_OIDC_WELL_KNOWN_CONFIG="op://keycloak/$APP_ENV/jwt-base/JWT_OIDC_WELL_KNOWN_CONFIG" diff --git a/strr-api/poetry.lock b/strr-api/poetry.lock index 5a16cabed..b8532255c 100644 --- a/strr-api/poetry.lock +++ b/strr-api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "alembic" @@ -312,8 +312,8 @@ files = [ [package.dependencies] cffi = [ - {version = ">=1.17.0", markers = "python_version >= \"3.13\""}, {version = ">=1.0.0", markers = "python_version < \"3.13\""}, + {version = ">=1.17.0", markers = "python_version >= \"3.13\""}, ] [[package]] @@ -1290,16 +1290,16 @@ files = [ google-auth = ">=2.14.1,<3.0.0" googleapis-common-protos = ">=1.56.3,<2.0.0" grpcio = [ - {version = ">=1.75.1,<2.0.0", optional = true, markers = "python_version >= \"3.14\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\" and python_version < \"3.14\""}, + {version = ">=1.75.1,<2.0.0", optional = true, markers = "python_version >= \"3.14\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.75.1,<2.0.0", optional = true, markers = "python_version >= \"3.14\" and extra == \"grpc\""}, {version = ">=1.49.1,<2.0.0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.75.1,<2.0.0", optional = true, markers = "python_version >= \"3.14\" and extra == \"grpc\""}, ] proto-plus = [ - {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, {version = ">=1.22.3,<2.0.0"}, + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, ] protobuf = ">=4.25.8,<7.0.0" requests = ">=2.20.0,<3.0.0" @@ -1376,8 +1376,8 @@ grpcio-status = ">=1.33.2" opentelemetry-api = ">=1.27.0" opentelemetry-sdk = ">=1.27.0" proto-plus = [ - {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, ] protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" diff --git a/strr-api/src/strr_api/config.py b/strr-api/src/strr_api/config.py index 2ce11b1c6..7e11e4dbb 100644 --- a/strr-api/src/strr_api/config.py +++ b/strr-api/src/strr_api/config.py @@ -45,11 +45,79 @@ import os from dotenv import find_dotenv, load_dotenv +from sqlalchemy.engine import URL basedir = os.path.abspath(os.path.dirname(__file__)) load_dotenv(find_dotenv()) +GCP_DEPLOYMENT_ENVS = {"development", "test", "uat", "sandbox", "production", "migration"} +PROXY_REQUIRED_ENVS = ("DATABASE_NAME",) + + +def _deployment_env() -> str: + return os.getenv("DEPLOYMENT_ENV", os.getenv("POD_NAMESPACE", "local")) + + +def _is_deployed_gcp() -> bool: + return bool(os.getenv("K_SERVICE")) or _deployment_env() in GCP_DEPLOYMENT_ENVS + + +def _use_proxy_iam() -> bool: + return _is_deployed_gcp() + + +def _cloudsql_user_env() -> str: + return "DATABASE_MIGRATION_USERNAME" if _deployment_env() == "migration" else "DATABASE_USERNAME" + + +def _require_proxy_env(user_env: str): + required = (*PROXY_REQUIRED_ENVS, user_env) + missing = [env_name for env_name in required if not os.getenv(env_name)] + if missing: + raise RuntimeError(f"Missing Cloud SQL IAM proxy environment variables: {', '.join(missing)}") + + +def _proxy_database_uri(user_env: str) -> str: + db_user = os.environ[user_env] + db_name = os.environ["DATABASE_NAME"] + db_host = os.getenv("DATABASE_HOST", "127.0.0.1") + db_port = int(os.getenv("DATABASE_PORT", "5432")) + + if db_unix_socket := os.getenv("DATABASE_UNIX_SOCKET", None): + return str( + URL.create( + "postgresql+psycopg2", + username=db_user, + database=db_name, + query={"host": db_unix_socket}, + ) + ) + + return str(URL.create("postgresql+psycopg2", username=db_user, host=db_host, port=db_port, database=db_name)) + + +def _local_database_uri() -> str: + db_user = os.getenv("DATABASE_USERNAME", "") + db_password = os.getenv("DATABASE_PASSWORD", "") + db_name = os.getenv("DATABASE_NAME", "") + db_host = os.getenv("DATABASE_HOST", "") + db_port = int(os.getenv("DATABASE_PORT", "5432")) + + if db_unix_socket := os.getenv("DATABASE_UNIX_SOCKET", None): + return f"postgresql+psycopg2://{db_user}:{db_password}@/{db_name}?host={db_unix_socket}" + + return f"postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db_name}" + + +def _database_settings() -> tuple[str, dict]: + if _use_proxy_iam(): + user_env = _cloudsql_user_env() + _require_proxy_env(user_env) + return _proxy_database_uri(user_env), {} + + return _local_database_uri(), {} + class Config: # pylint: disable=too-few-public-methods """Base class configuration that should set reasonable defaults. @@ -62,20 +130,10 @@ class Config: # pylint: disable=too-few-public-methods CSRF_ENABLED = True SECRET_KEY = "this-really-needs-to-be-changed" PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) - POD_NAMESPACE = os.getenv("DEPLOYMENT_ENV", "production") + POD_NAMESPACE = _deployment_env() SQLALCHEMY_TRACK_MODIFICATIONS = False - - DB_USER = os.getenv("DATABASE_USERNAME", "") - DB_PASSWORD = os.getenv("DATABASE_PASSWORD", "") - DB_NAME = os.getenv("DATABASE_NAME", "") - DB_HOST = os.getenv("DATABASE_HOST", "") - DB_PORT = int(os.getenv("DATABASE_PORT", "5432")) # POSTGRESQL - # POSTGRESQL - if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None): - SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg2://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?host={DB_UNIX_SOCKET}" - else: - SQLALCHEMY_DATABASE_URI = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}" + SQLALCHEMY_DATABASE_URI, SQLALCHEMY_ENGINE_OPTIONS = _database_settings() LD_SDK_KEY = os.getenv("LD_SDK_KEY", None) @@ -158,19 +216,7 @@ class Migration(Config): # pylint: disable=too-few-public-methods TESTING = False DEBUG = True - - # POSTGRESQL - DB_USER = os.getenv("DATABASE_USERNAME", "") - DB_PASSWORD = os.getenv("DATABASE_PASSWORD", "") - DB_NAME = os.getenv("DATABASE_NAME", "") - DB_HOST = os.getenv("DATABASE_HOST", "") - DB_PORT = int(os.getenv("DATABASE_PORT", "5432")) # POSTGRESQL - if DB_UNIX_SOCKET := os.getenv("DATABASE_UNIX_SOCKET", None): - SQLALCHEMY_DATABASE_URI = ( - f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@/{DB_NAME}?unix_sock={DB_UNIX_SOCKET}/.s.PGSQL.5432" - ) - else: - SQLALCHEMY_DATABASE_URI = f"postgresql+pg8000://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}" + SQLALCHEMY_DATABASE_URI, SQLALCHEMY_ENGINE_OPTIONS = _database_settings() class Testing(Config): # pylint: disable=too-few-public-methods @@ -188,6 +234,7 @@ class Testing(Config): # pylint: disable=too-few-public-methods f"postgresql://{DATABASE_TEST_USERNAME}:{DATABASE_TEST_PASSWORD}@" f"{DATABASE_TEST_HOST}:{DATABASE_TEST_PORT}/{DATABASE_TEST_NAME}" ) + SQLALCHEMY_ENGINE_OPTIONS = {} AUTH_SVC_URL = "https://test-auth-svc-url" PAYMENT_SVC_URL = "https://test-pay-url" diff --git a/strr-api/tests/integration/test_alembic_ownership.py b/strr-api/tests/integration/test_alembic_ownership.py index 61fa82fb3..c3ae9e9dd 100644 --- a/strr-api/tests/integration/test_alembic_ownership.py +++ b/strr-api/tests/integration/test_alembic_ownership.py @@ -2,17 +2,33 @@ from pathlib import Path +import docker +import pytest from alembic import command from alembic.config import Config +from docker.errors import DockerException from sqlalchemy import create_engine, text from testcontainers.postgres import PostgresContainer +def _require_docker(): + try: + client = docker.from_env() + client.ping() + except DockerException as exc: + pytest.skip(f"Docker is required for this integration test: {exc}") + finally: + if "client" in locals(): + client.close() + + def test_alembic_runs_with_configured_owner_role(monkeypatch): """Alembic runs migrations with the configured DB owner role.""" + _require_docker() + api_root = Path(__file__).resolve().parents[2] migrations_path = api_root / "migrations" - owner = "sa-api@bcrbk9-test.iam" + owner = "strr" with PostgresContainer("postgres:16-alpine") as postgres: db_url = postgres.get_connection_url() diff --git a/strr-api/tests/unit/test_config_cloudsql_iam.py b/strr-api/tests/unit/test_config_cloudsql_iam.py new file mode 100644 index 000000000..0f4e46083 --- /dev/null +++ b/strr-api/tests/unit/test_config_cloudsql_iam.py @@ -0,0 +1,106 @@ +"""Tests for strr-api Cloud SQL IAM database configuration.""" + +import importlib + +import dotenv +import pytest + +ENV_KEYS = ( + "DATABASE_HOST", + "DATABASE_MIGRATION_USERNAME", + "DATABASE_NAME", + "DATABASE_PASSWORD", + "DATABASE_PORT", + "DATABASE_UNIX_SOCKET", + "DATABASE_USERNAME", + "DEPLOYMENT_ENV", + "K_SERVICE", + "POD_NAMESPACE", +) + + +def _clear_env(monkeypatch): + for env_name in ENV_KEYS: + monkeypatch.delenv(env_name, raising=False) + + +def _reload_config(monkeypatch, **env): + _clear_env(monkeypatch) + # Keep these tests independent from each developer's local .env file. + monkeypatch.setattr(dotenv, "load_dotenv", lambda *_args, **_kwargs: False) + for env_name, value in env.items(): + monkeypatch.setenv(env_name, value) + + from strr_api import config as config_module + + return importlib.reload(config_module) + + +def test_production_config_uses_cloudsql_proxy_iam_uri(monkeypatch): + config_module = _reload_config( + monkeypatch, + DEPLOYMENT_ENV="production", + DATABASE_HOST="127.0.0.1", + DATABASE_NAME="strr-db", + DATABASE_PORT="5432", + DATABASE_USERNAME="sa-api@bcrbk9-prod.iam", + ) + + assert ( + config_module.Production.SQLALCHEMY_DATABASE_URI + == "postgresql+psycopg2://sa-api%40bcrbk9-prod.iam@127.0.0.1:5432/strr-db" + ) + assert config_module.Production.SQLALCHEMY_ENGINE_OPTIONS == {} + + +def test_migration_mode_uses_migration_iam_username(monkeypatch): + config_module = _reload_config( + monkeypatch, + DEPLOYMENT_ENV="migration", + DATABASE_HOST="127.0.0.1", + DATABASE_MIGRATION_USERNAME="sa-db-migrate@bcrbk9-dev.iam", + DATABASE_NAME="strr-db", + DATABASE_PORT="5432", + DATABASE_USERNAME="sa-api@bcrbk9-dev.iam", + ) + + assert ( + config_module.Migration.SQLALCHEMY_DATABASE_URI + == "postgresql+psycopg2://sa-db-migrate%40bcrbk9-dev.iam@127.0.0.1:5432/strr-db" + ) + + +def test_production_config_can_use_cloudsql_proxy_unix_socket(monkeypatch): + config_module = _reload_config( + monkeypatch, + DEPLOYMENT_ENV="production", + DATABASE_NAME="strr-db", + DATABASE_UNIX_SOCKET="/cloudsql/bcrbk9-dev:northamerica-northeast1:strr-db-dev", + DATABASE_USERNAME="sa-api@bcrbk9-dev.iam", + ) + + assert ( + config_module.Production.SQLALCHEMY_DATABASE_URI == "postgresql+psycopg2://sa-api%40bcrbk9-dev.iam@/strr-db" + "?host=%2Fcloudsql%2Fbcrbk9-dev%3Anorthamerica-northeast1%3Astrr-db-dev" + ) + assert config_module.Production.SQLALCHEMY_ENGINE_OPTIONS == {} + + +def test_deployed_config_requires_cloudsql_proxy_iam_env(monkeypatch): + with pytest.raises(RuntimeError, match="DATABASE_NAME, DATABASE_USERNAME"): + _reload_config(monkeypatch, DEPLOYMENT_ENV="production") + + +def test_local_config_keeps_password_database_uri(monkeypatch): + config_module = _reload_config( + monkeypatch, + DATABASE_HOST="localhost", + DATABASE_NAME="postgres", + DATABASE_PASSWORD="postgres", + DATABASE_PORT="15432", + DATABASE_USERNAME="postgres", + ) + + assert config_module.Production.POD_NAMESPACE == "local" + assert config_module.Production.SQLALCHEMY_DATABASE_URI == "postgresql://postgres:postgres@localhost:15432/postgres" + assert config_module.Production.SQLALCHEMY_ENGINE_OPTIONS == {} diff --git a/strr-api/tests/unit/test_gcp_iam_usage.py b/strr-api/tests/unit/test_gcp_iam_usage.py new file mode 100644 index 000000000..bac70b8f6 --- /dev/null +++ b/strr-api/tests/unit/test_gcp_iam_usage.py @@ -0,0 +1,84 @@ +# Copyright © 2023 Province of British Columbia +# +# Licensed under the BSD 3 Clause License, (the "License"); +# you may not use this file except in compliance with the License. +"""Checks that STRR deployments declare explicit GCP IAM identities.""" + +import re +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[3] + +SERVICE_ACCOUNT_RE = re.compile(r"^sa-(api|job)@(?P[-a-z0-9]+)\.iam\.gserviceaccount\.com$") + + +def _clouddeploy_files() -> list[Path]: + return sorted(path for path in REPO_ROOT.glob("**/devops/gcp/clouddeploy.yaml") if ".git" not in path.parts) + + +def _clouddeploy_stages(path: Path) -> list[dict[str, str]]: + clouddeploy = yaml.safe_load(path.read_text(encoding="utf-8")) + stages = [] + + for stage in clouddeploy.get("serialPipeline", {}).get("stages", []): + deploy_parameters = stage.get("deployParameters", []) + values = deploy_parameters[0].get("values", {}) if deploy_parameters else {} + stages.append({"target_id": stage.get("targetId"), **values}) + + return stages + + +def test_cloud_deploy_targets_use_explicit_runtime_service_accounts(): + """Cloud Run services/jobs should run as STRR IAM service accounts, not defaults.""" + clouddeploy_files = _clouddeploy_files() + + assert clouddeploy_files + + for clouddeploy_file in clouddeploy_files: + stages = _clouddeploy_stages(clouddeploy_file) + + assert stages, f"{clouddeploy_file} has no deployment stages" + + for stage in stages: + project_id = stage.get("deploy-project-id") + service_account = stage.get("service-account") + + assert project_id, f"{clouddeploy_file} {stage['target_id']} is missing deploy-project-id" + assert service_account, f"{clouddeploy_file} {stage['target_id']} is missing service-account" + assert SERVICE_ACCOUNT_RE.match( + service_account + ), f"{clouddeploy_file} {stage['target_id']} must use a named STRR IAM service account" + assert service_account.endswith( + f"@{project_id}.iam.gserviceaccount.com" + ), f"{clouddeploy_file} {stage['target_id']} service-account must match deploy-project-id" + + +def test_cd_workflows_use_workload_identity_deployment_secrets(): + """Deployment workflows should authenticate to GCP through workload identity.""" + cd_workflows = sorted((REPO_ROOT / ".github" / "workflows").glob("*-cd.y*ml")) + + assert cd_workflows + + for workflow in cd_workflows: + contents = workflow.read_text(encoding="utf-8") + + assert "WORKLOAD_IDENTIFY_POOLS_PROVIDER" in contents, f"{workflow} is missing workload identity provider" + assert "GCP_SERVICE_ACCOUNT" in contents, f"{workflow} is missing deployment service account" + + +def test_migrated_strr_api_vault_mapping_uses_cloudsql_iam(): + """The migrated API deployment should use passwordless DB access through the proxy.""" + contents = (REPO_ROOT / "strr-api" / "devops" / "vaults.gcp.env").read_text(encoding="utf-8") + + for forbidden_env in ("DATABASE_PASSWORD=", "DATABASE_UNIX_SOCKET="): + assert forbidden_env not in contents + + for required_env in ( + 'DATABASE_HOST="127.0.0.1"', + "DATABASE_MIGRATION_USERNAME=", + 'DATABASE_PORT="5432"', + "DATABASE_USERNAME=", + ): + assert required_env in contents