Move strr-api deployed DB access to Cloud SQL IAM auth - #1763
Conversation
|
Temporary Url for review: https://strr-hosts-dev--pr-1763-ciu8zc70.web.app |
bc568e9 to
1834cc0
Compare
1834cc0 to
16ef0a0
Compare
| return f"postgresql+pg8000://{db_user}:{db_password}@/{db_name}?unix_sock={db_unix_socket}/.s.PGSQL.5432" | ||
| return f"postgresql+psycopg2://{db_user}:{db_password}@/{db_name}?host={db_unix_socket}" | ||
|
|
||
| if driver == "pg8000": |
There was a problem hiding this comment.
why are switching to pg8000 from psycopg2. lets use auth proxy from gcp for the iam auth.
ref: https://docs.cloud.google.com/sql/docs/postgres/sql-proxy
There was a problem hiding this comment.
I had thought about this but figured it would be a more complex change and more resistance. I can switch to proxy
There was a problem hiding this comment.
switched to proxy please review
| assert config_module.Production.SQLALCHEMY_ENGINE_OPTIONS == {} | ||
|
|
||
|
|
||
| def test_deployed_config_requires_cloudsql_proxy_iam_env(monkeypatch): |
There was a problem hiding this comment.
this test case is failing for me in local. do i have configure the .env in a certain way?
There was a problem hiding this comment.
sorry you shouldn’t need to configure your .env for this test. I reproduced the failure with a normal local .env and the config reload was loading those values after the test cleared the environment, so the RuntimeError was not raised. I updated the test helper to disable dotenv loading
|
| GCP_DEPLOYMENT_ENVS = {"development", "test", "uat", "sandbox", "production", "migration"} | ||
| PROXY_REQUIRED_ENVS = ("DATABASE_NAME",) | ||
|
|
||
|
|
||
| def _deployment_env() -> str: | ||
| return os.getenv("DEPLOYMENT_ENV", os.getenv("POD_NAMESPACE", "local")) | ||
|
|
||
|
|
||
| def _is_deployed_gcp() -> bool: | ||
| return bool(os.getenv("K_SERVICE")) or _deployment_env() in GCP_DEPLOYMENT_ENVS | ||
|
|
||
|
|
||
| def _use_proxy_iam() -> bool: | ||
| return _is_deployed_gcp() | ||
|
|
||
|
|
||
| def _cloudsql_user_env() -> str: | ||
| return "DATABASE_MIGRATION_USERNAME" if _deployment_env() == "migration" else "DATABASE_USERNAME" | ||
|
|
||
|
|
||
| def _require_proxy_env(user_env: str): | ||
| required = (*PROXY_REQUIRED_ENVS, user_env) | ||
| missing = [env_name for env_name in required if not os.getenv(env_name)] | ||
| if missing: | ||
| raise RuntimeError(f"Missing Cloud SQL IAM proxy environment variables: {', '.join(missing)}") | ||
|
|
||
|
|
||
| def _proxy_database_uri(user_env: str) -> str: | ||
| db_user = os.environ[user_env] | ||
| db_name = os.environ["DATABASE_NAME"] | ||
| db_host = os.getenv("DATABASE_HOST", "127.0.0.1") | ||
| db_port = int(os.getenv("DATABASE_PORT", "5432")) | ||
|
|
||
| if db_unix_socket := os.getenv("DATABASE_UNIX_SOCKET", None): | ||
| return str( | ||
| URL.create( | ||
| "postgresql+psycopg2", | ||
| username=db_user, | ||
| database=db_name, | ||
| query={"host": db_unix_socket}, | ||
| ) | ||
| ) | ||
|
|
||
| return str(URL.create("postgresql+psycopg2", username=db_user, host=db_host, port=db_port, database=db_name)) | ||
|
|
||
|
|
||
| def _local_database_uri() -> str: | ||
| db_user = os.getenv("DATABASE_USERNAME", "") | ||
| db_password = os.getenv("DATABASE_PASSWORD", "") | ||
| db_name = os.getenv("DATABASE_NAME", "") | ||
| db_host = os.getenv("DATABASE_HOST", "") | ||
| db_port = int(os.getenv("DATABASE_PORT", "5432")) | ||
|
|
||
| if db_unix_socket := os.getenv("DATABASE_UNIX_SOCKET", None): | ||
| return f"postgresql+psycopg2://{db_user}:{db_password}@/{db_name}?host={db_unix_socket}" | ||
|
|
||
| return f"postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db_name}" | ||
|
|
||
|
|
||
| def _database_settings() -> tuple[str, dict]: | ||
| if _use_proxy_iam(): | ||
| user_env = _cloudsql_user_env() | ||
| _require_proxy_env(user_env) | ||
| return _proxy_database_uri(user_env), {} | ||
|
|
||
| return _local_database_uri(), {} | ||
|
|
There was a problem hiding this comment.
The changes you made in this PR should work here as well right? we could remove all these lines and simplify
3 participants
Summary
This moves deployed
strr-apidatabase access from password/socket DB auth to Cloud SQL IAM database authentication through the Cloud SQL Auth Proxy.Local Docker and pytest still use the existing password-based Postgres config. Normal local dev is unaffected as long as local env stays on
POD_NAMESPACE=localor leavesDEPLOYMENT_ENVunset.Changes
cloud-sql-python-connector[pg8000]and kept the normalpsycopg2SQLAlchemy path.strr-apiconfig to build a passwordlesspostgresql+psycopg2://URI for the proxy endpoint.DATABASE_USERNAME; migration mode usesDATABASE_MIGRATION_USERNAMEwhenDEPLOYMENT_ENV=migration.DATABASE_HOST=127.0.0.1andDATABASE_PORT=5432for the proxy sidecar.mainthroughDATABASE_OWNER_ROLE.Deployment Handoff
I do not have access to update these 1Password values directly, so this still needs to be handled by someone with access.
Needed STRR GCP env mapping changes:
Remove deployed use of:
DATABASE_PASSWORDDATABASE_UNIX_SOCKETCLOUDSQL_INSTANCE_CONNECTION_NAMECLOUDSQL_IP_TYPEAdd/keep:
DATABASE_NAME=strr-dbDATABASE_HOST=127.0.0.1DATABASE_PORT=5432DATABASE_USERNAMEDATABASE_MIGRATION_USERNAMEDATABASE_OWNER_ROLE=strrFor UAT, use the
bcrbk9-testidentity and Cloud SQL instance because UAT deploys into the test project.The Cloud Run deployment layer also needs to run Cloud SQL Auth Proxy v2 with automatic IAM database authentication, listening on
127.0.0.1:5432, using the existingcloudsql-instancesvalue for the instance connection name: