From 9e4760d6ef2d2ab2e36cbeffb84903edfc0e211b Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:20:56 -0400 Subject: [PATCH 1/7] fix async concurrent and add surface tests --- .github/workflows/build-test.yml | 76 +++++- docs/OperationsAPI.yaml | 13 +- .../impl/feeds_operations_impl.py | 62 +++-- .../impl/licenses_api_impl.py | 8 +- .../impl/request_validator.py | 4 +- .../test_create_feeds_operations_impl_gtfs.py | 10 +- .../impl/test_feeds_operations_impl_gtfs.py | 16 +- .../test_feeds_operations_impl_gtfs_rt.py | 12 +- .../impl/test_licenses_api_impl.py | 24 +- .../impl/test_licenses_propagate_match.py | 20 +- .../tests/test_routes_are_sync.py | 69 +++++ scripts/api-operations-surface.sh | 247 ++++++++++++++++++ 12 files changed, 486 insertions(+), 75 deletions(-) create mode 100644 functions-python/operations_api/tests/test_routes_are_sync.py create mode 100755 scripts/api-operations-surface.sh diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1d9aa9009..b5f945209 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -287,4 +287,78 @@ jobs: with: name: coverage_report_${{ matrix.group }} path: scripts/coverage_reports/ - overwrite: true \ No newline at end of file + overwrite: true + + operations-api-surface: + needs: setup + if: ${{ needs.setup.outputs.should-run-tests == 'true' }} + runs-on: ubuntu-latest + name: Operations API surface + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: ${{ env.python_version }} + + - name: Download Operations API generated code + uses: actions/download-artifact@v8 + with: + name: feeds_operations_gen + path: functions-python/operations_api/src/feeds_gen/ + + - name: Docker Compose DB + run: docker compose --env-file ./config/.env.local up -d postgres-test --wait + working-directory: ${{ github.workspace }} + + - name: Set up JDK ${{ env.java_version }} + uses: actions/setup-java@v5 + with: + java-version: ${{ env.java_version }} + distribution: 'temurin' + + - name: Cache Liquibase + id: cache-liquibase + uses: actions/cache@v6 + with: + path: ~/.liquibase + key: liquibase-${{ env.liquibase_version }} + + - name: Install Liquibase + if: steps.cache-liquibase.outputs.cache-hit != 'true' + env: + LIQUIBASE_VERSION: ${{ env.liquibase_version }} + run: | + curl -sSL https://github.com/liquibase/liquibase/releases/download/v${LIQUIBASE_VERSION}/liquibase-${LIQUIBASE_VERSION}.tar.gz -o liquibase.tar.gz + rm -rf liquibase-dist + mkdir liquibase-dist + tar -xzf liquibase.tar.gz -C liquibase-dist + mv liquibase-dist ~/.liquibase + + - name: Link Liquibase + run: sudo ln -sf ~/.liquibase/liquibase /usr/local/bin/liquibase + + - name: Run Liquibase on test DB + working-directory: ${{ github.workspace }}/liquibase + run: | + export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog.xml" + export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:54320/MobilityDatabaseTest + export LIQUIBASE_COMMAND_USERNAME=postgres + export LIQUIBASE_COMMAND_PASSWORD=postgres + liquibase update + + - name: Run Liquibase on users test DB + working-directory: ${{ github.workspace }}/liquibase + run: | + export LIQUIBASE_COMMAND_CHANGELOG_FILE="changelog_user.xml" + export LIQUIBASE_COMMAND_URL=jdbc:postgresql://localhost:54320/${POSTGRES_USER_TEST_DB:-MobilityDatabaseUsersTest} + export LIQUIBASE_COMMAND_USERNAME=postgres + export LIQUIBASE_COMMAND_PASSWORD=postgres + liquibase update + + # DB is already up + migrated by the steps above, so run with --no-docker. + # The script provisions the venv/deps/symlinks, starts the API, and hits every endpoint. + - name: Run Operations API surface suite + shell: bash + run: scripts/api-operations-surface.sh --no-docker \ No newline at end of file diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index 538f2cef7..636a0627b 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -979,6 +979,7 @@ components: example: vp description: > The type of realtime entry: + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -1095,6 +1096,7 @@ components: example: vp description: > The type of realtime entry: + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -1332,21 +1334,18 @@ components: properties: producer_url: description: > - URL where the producer is providing the dataset. Refer to the authentication information to know how to access this URL. + URL where the producer is providing the dataset. Refer to the authentication information to know how to access this URL. type: string format: url example: https://ladotbus.com/gtfs is_producer_url_unstable: description: > - Indicates whether the `producer_url` is known to be unstable, i.e. it changes over time. - This may be because the URL contains a date/time, or because the transit provider has - communicated that it is not permanent (e.g. it is updated monthly). + Indicates whether the `producer_url` is known to be unstable, i.e. it changes over time. This may be because the URL contains a date/time, or because the transit provider has communicated that it is not permanent (e.g. it is updated monthly). * true - The producer URL is unstable and changes over time. * false - The producer URL is stable and unchanging over time. * null (default) - There is not enough information to determine the stability of the producer URL. - type: boolean nullable: true example: true @@ -1956,6 +1955,7 @@ components: example: vp description: > The type of realtime entry: + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2080,6 +2080,7 @@ components: example: vp description: > The type of realtime entry: + * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2167,6 +2168,7 @@ components: x-operation: true description: > Describes status of the Feed. Should be one of + * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -2184,6 +2186,7 @@ components: x-operation: true description: > Describes data type of a feed. Should be one of + * `gtfs` GTFS feed. * `gtfs_rt` GTFS-RT feed. * `gbfs` GBFS feed. diff --git a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py index 27d612c10..4445bae21 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py @@ -18,6 +18,7 @@ from fastapi.responses import JSONResponse import logging +import os from datetime import datetime from typing import Annotated, Optional @@ -88,6 +89,18 @@ from .models.operation_gtfs_feed_impl import OperationGtfsFeedImpl from .models.operation_gtfs_rt_feed_impl import OperationGtfsRtFeedImpl from .request_validator import validate_request +from shared.helpers.transform import to_boolean + + +def _external_side_effects_enabled() -> bool: + """Whether to invoke external GCP side-effects (Pub/Sub dataset download, + Cloud Tasks revalidation, notifications). + + Skipped when running locally (``LOCAL_ENV=True``) so that create/update never + construct or call those clients outside a real deployment — this keeps local + runs and tests fast and free of live side-effects. + """ + return not to_boolean(os.getenv("LOCAL_ENV", False)) class OperationsApiImpl(BaseOperationsApi): @@ -296,7 +309,7 @@ def detect_changes( return diff @validate_request(UpdateRequestGtfsFeed, "update_request_gtfs_feed") - async def update_gtfs_feed( + def update_gtfs_feed( self, update_request_gtfs_feed: Annotated[ UpdateRequestGtfsFeed, @@ -310,10 +323,10 @@ async def update_gtfs_feed( - 400: Feed ID not found. - 500: Internal server error. """ - return await self._update_feed(update_request_gtfs_feed, DataType.GTFS) + return self._update_feed(update_request_gtfs_feed, DataType.GTFS) @validate_request(UpdateRequestGtfsRtFeed, "update_request_gtfs_rt_feed") - async def update_gtfs_rt_feed( + def update_gtfs_rt_feed( self, update_request_gtfs_rt_feed: Annotated[ UpdateRequestGtfsRtFeed, @@ -327,10 +340,10 @@ async def update_gtfs_rt_feed( - 400: Feed ID not found. - 500: Internal server error. """ - return await self._update_feed(update_request_gtfs_rt_feed, DataType.GTFS_RT) + return self._update_feed(update_request_gtfs_rt_feed, DataType.GTFS_RT) @with_db_session - async def _update_feed( + def _update_feed( self, update_request_feed: UpdateRequestGtfsFeed | UpdateRequestGtfsRtFeed, data_type: DataType, @@ -340,7 +353,7 @@ async def _update_feed( Update the specified feed in the Mobility Database """ try: - feed_from_db = await OperationsApiImpl.fetch_feed( + feed_from_db = OperationsApiImpl.fetch_feed( data_type, db_session, update_request_feed ) @@ -365,7 +378,7 @@ async def _update_feed( r.target_id for r in getattr(feed_from_db, "redirectingids", []) } - await OperationsApiImpl._populate_feed_values( + OperationsApiImpl._populate_feed_values( feed_from_db, impl_class, db_session, update_request_feed ) if getattr(update_request_feed, "propagate_license", False): @@ -394,6 +407,10 @@ async def _update_feed( update_request_feed.id, diff.values(), ) + # Skip external notifications / revalidation (Pub/Sub, Cloud Tasks) + # when running locally so no live clients are constructed or called. + if not _external_side_effects_enabled(): + return Response(status_code=200) # Emit notification events for URL / redirect changes (best-effort, fire-and-forget). feed_stable_id = update_request_feed.id new_producer_url = getattr(feed_from_db, "producer_url", None) @@ -459,7 +476,7 @@ async def _update_feed( raise HTTPException(status_code=500, detail=f"Internal server error: {e}") @staticmethod - async def _populate_feed_values(feed, impl_class, session, update_request_feed): + def _populate_feed_values(feed, impl_class, session, update_request_feed): impl_class.to_orm(update_request_feed, feed, session) action = update_request_feed.operational_status_action # This is a temporary solution as the operational_status is not visible in the diff @@ -473,7 +490,7 @@ async def _populate_feed_values(feed, impl_class, session, update_request_feed): session.add(feed) @staticmethod - async def fetch_feed(data_type, session, update_request_feed): + def fetch_feed(data_type, session, update_request_feed): """Fetch a feed by its stable ID with eager loading. Args: @@ -499,7 +516,7 @@ async def fetch_feed(data_type, session, update_request_feed): return feed @with_db_session - async def create_gtfs_feed( + def create_gtfs_feed( self, operation_create_request_gtfs_feed: Annotated[ OperationCreateRequestGtfsFeed, @@ -541,17 +558,18 @@ async def create_gtfs_feed( else: assign_license_by_url(created_feed, db_session) db_session.commit() - try: - trigger_dataset_download( - created_feed, - get_execution_id(get_request_context(), "feed-created-process"), - ) - except Exception as exc: - logging.error( - "Failed to trigger dataset download for feed ID: %s. Error: %s", - created_feed.stable_id, - exc, - ) + if _external_side_effects_enabled(): + try: + trigger_dataset_download( + created_feed, + get_execution_id(get_request_context(), "feed-created-process"), + ) + except Exception as exc: + logging.error( + "Failed to trigger dataset download for feed ID: %s. Error: %s", + created_feed.stable_id, + exc, + ) logging.info("Created new GTFS feed with ID: %s", new_feed.stable_id) refreshed = refresh_materialized_view(db_session, t_feedsearch.name) logging.info("Materialized view %s refreshed: %s", t_feedsearch.name, refreshed) @@ -559,7 +577,7 @@ async def create_gtfs_feed( return JSONResponse(status_code=201, content=jsonable_encoder(payload)) @with_db_session - async def create_gtfs_rt_feed( + def create_gtfs_rt_feed( self, operation_create_request_gtfs_rt_feed: Annotated[ OperationCreateRequestGtfsRtFeed, diff --git a/functions-python/operations_api/src/feeds_operations/impl/licenses_api_impl.py b/functions-python/operations_api/src/feeds_operations/impl/licenses_api_impl.py index 389696e01..67cce1186 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/licenses_api_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/licenses_api_impl.py @@ -68,7 +68,7 @@ def handle_get_license( # Build Pydantic model from ORM object attributes return LicenseWithRulesImpl.from_orm(license_orm) - async def get_license( + def get_license( self, id: StrictStr, ) -> LicenseWithRules: @@ -136,7 +136,7 @@ def handle_get_licenses( return [LicenseBaseImpl.from_orm(item) for item in items] - async def get_licenses( + def get_licenses( self, offset: str = "0", limit: str = "100", @@ -173,7 +173,7 @@ def handle_get_matching_licenses( status_code=500, detail="Error retrieving matching licenses" ) - async def get_matching_licenses( + def get_matching_licenses( self, get_matching_licenses_request: GetMatchingLicensesRequest, ) -> List[MatchingLicense]: @@ -222,7 +222,7 @@ def handle_propagate_match_license( logging.error("Error in propagate_match_license: %s", e) raise HTTPException(status_code=500, detail="Internal server error") - async def propagate_match_license( + def propagate_match_license( self, propagate_license_request: PropagateLicenseRequest, ) -> PropagateLicenseResponse: diff --git a/functions-python/operations_api/src/feeds_operations/impl/request_validator.py b/functions-python/operations_api/src/feeds_operations/impl/request_validator.py index 95dd10a11..a206ecbb0 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/request_validator.py +++ b/functions-python/operations_api/src/feeds_operations/impl/request_validator.py @@ -29,7 +29,7 @@ def validate_request(model: BaseModel, parameter_name: str, validate_none: bool def decorator(func): @wraps(func) - async def wrapper(*args, **kwargs): + def wrapper(*args, **kwargs): func_args = inspect.getfullargspec(func).args print(func_args) index = func_args.index(parameter_name) @@ -42,7 +42,7 @@ async def wrapper(*args, **kwargs): else: if validate_none: raise HTTPException(status_code=400, detail="Missing parameter") - return await func(*args, **kwargs) + return func(*args, **kwargs) return wrapper diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py b/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py index ef71437fa..e9d2e51ac 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py @@ -91,7 +91,7 @@ async def test_create_gtfs_feed_success(_, mock_publish_messages, db_session): related_links=[], ) - response = await api.create_gtfs_feed(request) + response = api.create_gtfs_feed(request) assert response.status_code == 201 payload = json.loads(response.body) @@ -163,7 +163,7 @@ async def test_create_gtfs_feed_duplicate_url_rejected(_): ) with pytest.raises(HTTPException) as exc_info: - await api.create_gtfs_feed(request) + api.create_gtfs_feed(request) assert exc_info.value.status_code == 400 assert ( @@ -205,7 +205,7 @@ async def test_create_gtfs_rt_feed_success(_, db_session): entity_types=["vp", "tu"], ) - response = await api.create_gtfs_rt_feed(request) + response = api.create_gtfs_rt_feed(request) assert response.status_code == 201 payload = json.loads(response.body) @@ -288,7 +288,7 @@ def fake_propagate(license_id, license_url, db_session, **kwargs): propagate_license=True, ) - response = await api.create_gtfs_feed(request) + response = api.create_gtfs_feed(request) payload = json.loads(response.body) try: @@ -331,7 +331,7 @@ async def test_create_gtfs_rt_feed_duplicate_url_rejected(_): ) with pytest.raises(HTTPException) as exc_info: - await api.create_gtfs_rt_feed(request) + api.create_gtfs_rt_feed(request) assert exc_info.value.status_code == 400 assert ( diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py index 78f5c7fd8..03d51c21a 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs.py @@ -52,7 +52,7 @@ def db_session(): @patch("feeds_operations.impl.feeds_operations_impl.create_web_revalidation_task") async def test_update_gtfs_feed_no_changes(mock_revalidation, update_request_gtfs_feed): api = OperationsApiImpl() - response: Response = await api.update_gtfs_feed(update_request_gtfs_feed) + response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 200 @@ -70,7 +70,7 @@ async def test_update_gtfs_feed_field_change( ) ] api = OperationsApiImpl() - response: Response = await api.update_gtfs_feed(update_request_gtfs_feed) + response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 200 db_feed = ( @@ -89,7 +89,7 @@ async def test_update_gtfs_feed_set_wip( ): update_request_gtfs_feed.operational_status_action = "wip" api = OperationsApiImpl() - response: Response = await api.update_gtfs_feed(update_request_gtfs_feed) + response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 200 db_feed = ( @@ -105,7 +105,7 @@ async def test_update_gtfs_feed_set_wip( async def test_update_gtfs_feed_set_wip_nochange(update_request_gtfs_feed, db_session): update_request_gtfs_feed.operational_status_action = "no_change" api = OperationsApiImpl() - response: Response = await api.update_gtfs_feed(update_request_gtfs_feed) + response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 204 db_feed = ( @@ -124,7 +124,7 @@ async def test_update_gtfs_feed_set_published( ): update_request_gtfs_feed.operational_status_action = "published" api = OperationsApiImpl() - response: Response = await api.update_gtfs_feed(update_request_gtfs_feed) + response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 200 db_feed = ( @@ -143,7 +143,7 @@ async def test_update_gtfs_feed_set_unpublished( ): update_request_gtfs_feed.operational_status_action = "unpublished" api = OperationsApiImpl() - response: Response = await api.update_gtfs_feed(update_request_gtfs_feed) + response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 200 db_feed = ( @@ -159,7 +159,7 @@ async def test_update_gtfs_feed_invalid_feed(update_request_gtfs_feed): update_request_gtfs_feed.id = "invalid" api = OperationsApiImpl() with pytest.raises(HTTPException) as exc_info: - await api.update_gtfs_feed(update_request_gtfs_feed) + api.update_gtfs_feed(update_request_gtfs_feed) assert exc_info.value.status_code == 400 assert exc_info.value.detail == "Feed ID not found: invalid" @@ -169,7 +169,7 @@ async def test_update_gtfs_feed_official_field(update_request_gtfs_feed, db_sess """Test updating the official field of a GTFS feed.""" update_request_gtfs_feed.official = True api = OperationsApiImpl() - response: Response = await api.update_gtfs_feed(update_request_gtfs_feed) + response: Response = api.update_gtfs_feed(update_request_gtfs_feed) assert response.status_code == 204 db_feed = ( diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py index 5a8b857dd..94f9f4911 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py @@ -54,7 +54,7 @@ async def test_update_gtfs_feed_field_change( ): update_request_gtfs_rt_feed.feed_name = "New feed name" api = OperationsApiImpl() - response: Response = await api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) + response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 db_feed = ( @@ -73,7 +73,7 @@ async def test_update_gtfs_feed_static_change( ): update_request_gtfs_rt_feed.feed_references = ["mdb-400"] api = OperationsApiImpl() - response: Response = await api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) + response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 db_feed = ( @@ -96,7 +96,7 @@ async def test_update_gtfs_rt_feed_set_wip( ): update_request_gtfs_rt_feed.operational_status_action = "wip" api = OperationsApiImpl() - response: Response = await api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) + response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 db_feed = ( @@ -115,7 +115,7 @@ async def test_update_gtfs_rt_feed_set_published( ): update_request_gtfs_rt_feed.operational_status_action = "published" api = OperationsApiImpl() - response: Response = await api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) + response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 db_feed = ( @@ -134,7 +134,7 @@ async def test_update_gtfs_rt_feed_set_unpublished( ): update_request_gtfs_rt_feed.operational_status_action = "unpublished" api = OperationsApiImpl() - response: Response = await api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) + response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 db_feed = ( @@ -153,7 +153,7 @@ async def test_update_gtfs_rt_feed_official_field( """Test updating the official field of a GTFS-RT feed.""" update_request_gtfs_rt_feed.official = True api = OperationsApiImpl() - response: Response = await api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) + response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 db_feed = ( diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_licenses_api_impl.py b/functions-python/operations_api/tests/feeds_operations/impl/test_licenses_api_impl.py index fea886bac..62189ba8f 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_licenses_api_impl.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_licenses_api_impl.py @@ -28,7 +28,7 @@ async def test_get_license_success(db_session): api = LicensesApiImpl() # Act - result = await api.get_license(existing.id) + result = api.get_license(existing.id) # Assert assert isinstance(result, LicenseWithRules) @@ -41,7 +41,7 @@ async def test_get_license_not_found(): api = LicensesApiImpl() with pytest.raises(HTTPException) as exc_info: - await api.get_license("non-existent-license-id") + api.get_license("non-existent-license-id") assert exc_info.value.status_code == 404 assert exc_info.value.detail == "License not found" @@ -57,7 +57,7 @@ async def test_get_licenses_basic_pagination(db_session): api = LicensesApiImpl() # Act - result = await api.get_licenses(limit=10, offset=0) + result = api.get_licenses(limit=10, offset=0) # Assert assert isinstance(result, list) @@ -77,7 +77,7 @@ async def test_get_licenses_with_search_query(db_session): fragment = existing.name[:3] api = LicensesApiImpl() - result = await api.get_licenses(limit=50, offset=0, search_query=fragment) + result = api.get_licenses(limit=50, offset=0, search_query=fragment) assert isinstance(result, list) # True positive: the reference license must be present @@ -101,7 +101,7 @@ async def test_get_licenses_filter_is_spdx_true(db_session): pytest.skip("No SPDX licenses in test database to validate is_spdx filter") api = LicensesApiImpl() - result = await api.get_licenses(limit=10, offset=0, is_spdx=True) + result = api.get_licenses(limit=10, offset=0, is_spdx=True) assert isinstance(result, list) assert len(result) == spdx_count @@ -118,7 +118,7 @@ async def test_get_licenses_filter_is_spdx_false(db_session): pytest.skip("No non-SPDX licenses in test database to validate is_spdx filter") api = LicensesApiImpl() - result = await api.get_licenses(limit=10, offset=0, is_spdx=False) + result = api.get_licenses(limit=10, offset=0, is_spdx=False) assert isinstance(result, list) assert len(result) == non_spdx_count @@ -137,7 +137,7 @@ async def test_get_license_includes_rules(db_session): pytest.skip("Fixture license 'custom-test' not found in test database") api = LicensesApiImpl() - result = await api.get_license("custom-test") + result = api.get_license("custom-test") assert isinstance(result, LicenseWithRules) # LicenseWithRules exposes rules under the license_rules field @@ -150,12 +150,12 @@ async def test_get_license_includes_rules(db_session): async def test_get_licenses_includes_rules_for_each_item(db_session): """Ensure that licenses returned by get_licenses can expose rule data via get_license.""" api = LicensesApiImpl() - results = await api.get_licenses(limit=10, offset=0) + results = api.get_licenses(limit=10, offset=0) assert results for item in results: if item.id == "MIT": - detailed = await api.get_license("MIT") + detailed = api.get_license("MIT") assert isinstance(detailed, LicenseWithRules) assert detailed.license_rules is not None assert [r.name for r in detailed.license_rules] == ["attribution"] @@ -187,7 +187,7 @@ def fake_resolve(license_url, db_session): # signature matches resolve_license ) # Act - result = await api.get_matching_licenses(request) + result = api.get_matching_licenses(request) # Assert assert isinstance(result, list) @@ -211,7 +211,7 @@ def fake_resolve(license_url, db_session): ) with pytest.raises(HTTPException) as exc_info: - await api.get_matching_licenses(request) + api.get_matching_licenses(request) assert exc_info.value.status_code == 500 assert exc_info.value.detail == "Error retrieving matching licenses" @@ -232,7 +232,7 @@ def fake_resolve(license_url, db_session): ) with pytest.raises(HTTPException) as exc_info: - await api.get_matching_licenses(request) + api.get_matching_licenses(request) assert exc_info.value.status_code == 500 assert exc_info.value.detail == "Error retrieving matching licenses" diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_licenses_propagate_match.py b/functions-python/operations_api/tests/feeds_operations/impl/test_licenses_propagate_match.py index 68b832060..7455aec76 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_licenses_propagate_match.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_licenses_propagate_match.py @@ -72,7 +72,7 @@ async def test_propagate_match_license_dry_run_returns_response(monkeypatch): dry_run=True, override=False, ) - result = await api.propagate_match_license(request) + result = api.propagate_match_license(request) assert isinstance(result, PropagateLicenseResponse) assert result.dry_run is True @@ -99,7 +99,7 @@ async def test_propagate_match_license_not_dry_run_returns_response(monkeypatch) dry_run=False, override=False, ) - result = await api.propagate_match_license(request) + result = api.propagate_match_license(request) assert isinstance(result, PropagateLicenseResponse) assert result.dry_run is False @@ -125,7 +125,7 @@ def raise_value_error(**kwargs): ) with pytest.raises(HTTPException) as exc_info: - await api.propagate_match_license(request) + api.propagate_match_license(request) assert exc_info.value.status_code == 400 assert "BAD" in exc_info.value.detail @@ -151,7 +151,7 @@ def raise_runtime(**kwargs): ) with pytest.raises(HTTPException) as exc_info: - await api.propagate_match_license(request) + api.propagate_match_license(request) assert exc_info.value.status_code == 500 @@ -181,7 +181,7 @@ async def test_propagate_match_license_override_false_returns_only_null_feeds( dry_run=True, override=False, ) - result = await api.propagate_match_license(request) + result = api.propagate_match_license(request) assert result.total_feeds_with_same_url == 3 assert result.affected_feeds_count == 1 @@ -215,7 +215,7 @@ async def test_propagate_match_license_override_true_includes_existing_license_f dry_run=True, override=True, ) - result = await api.propagate_match_license(request) + result = api.propagate_match_license(request) assert result.override is True assert result.affected_feeds_count == 2 @@ -239,7 +239,7 @@ async def test_propagate_match_license_db_unknown_license_raises_400(db_session) ) with pytest.raises(HTTPException) as exc_info: - await api.propagate_match_license(request) + api.propagate_match_license(request) assert exc_info.value.status_code == 400 @@ -267,7 +267,7 @@ async def test_propagate_match_license_dry_run_no_db_changes(db_session): dry_run=True, ) - result = await api.propagate_match_license(request) + result = api.propagate_match_license(request) db_session.expire(feed) assert feed.license_id == before_license_id, "dry_run=True must not change the DB" @@ -306,7 +306,7 @@ async def test_propagate_match_license_applies_changes_when_not_dry_run(db_sessi dry_run=False, override=False, ) - result = await api.propagate_match_license(request) + result = api.propagate_match_license(request) db_session.expire_all() @@ -365,7 +365,7 @@ async def test_propagate_match_license_override_false_skips_feeds_with_license( dry_run=False, override=False, ) - result = await api.propagate_match_license(request) + result = api.propagate_match_license(request) db_session.expire(feed) assert ( diff --git a/functions-python/operations_api/tests/test_routes_are_sync.py b/functions-python/operations_api/tests/test_routes_are_sync.py new file mode 100644 index 000000000..b94d48fda --- /dev/null +++ b/functions-python/operations_api/tests/test_routes_are_sync.py @@ -0,0 +1,69 @@ +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +"""Structural guard against the async/not-awaited route bug. + +The generated FastAPI routers invoke the impl methods WITHOUT `await` +(``return BaseXApi.subclasses[0]().(...)``). If an impl route method is +declared ``async`` it therefore returns an un-awaited coroutine, which FastAPI +cannot serialize -> HTTP 500 (ResponseValidationError) at runtime. + +The unit tests call the impls directly and ``await`` them, so they never catch +this. This test asserts, purely by reflection (no DB, no server, milliseconds), +that every route-backing impl method -- and anything it wraps -- is synchronous. +""" +import inspect + +import pytest + +# Base classes declare the route methods; importing the impl modules registers +# each impl into ``Base.subclasses`` via ``__init_subclass__``. +from feeds_gen.apis.operations_api_base import BaseOperationsApi +from feeds_gen.apis.licenses_api_base import BaseLicensesApi +from feeds_gen.apis.users_api_base import BaseUsersApi +from feeds_operations.impl.feeds_operations_impl import OperationsApiImpl # noqa: F401 +from feeds_operations.impl.licenses_api_impl import LicensesApiImpl # noqa: F401 +from feeds_operations.impl.user_feature_flags_impl import UserFeatureFlagsApiImpl # noqa: F401 + +BASES = (BaseOperationsApi, BaseLicensesApi, BaseUsersApi) + + +def _route_methods(): + """Yield (impl_class, method_name) for every route-backing method.""" + for base in BASES: + assert base.subclasses, f"No impl registered for {base.__name__}" + impl = base.subclasses[0] + for name, _ in inspect.getmembers(base, predicate=inspect.isfunction): + if name.startswith("_"): + continue + yield impl, name + + +CASES = list(_route_methods()) + + +def test_all_routes_discovered(): + # Sanity: all three routers contribute methods (guards against an import/registration regression). + assert len(CASES) >= 19, f"Expected >= 19 route methods, found {len(CASES)}" + + +@pytest.mark.parametrize( + "impl, method_name", + CASES, + ids=[f"{impl.__name__}.{name}" for impl, name in CASES], +) +def test_route_method_is_synchronous(impl, method_name): + """A route method (and any function it wraps) must be a plain `def`.""" + fn = getattr(impl, method_name) + seen = set() + while fn is not None and id(fn) not in seen: + seen.add(id(fn)) + assert not inspect.iscoroutinefunction(fn), ( + f"{impl.__name__}.{method_name} (or a function it wraps) is `async`. " + "The generated router calls it without `await`, so it returns an " + "un-awaited coroutine at runtime (HTTP 500). Make it a regular `def`." + ) + fn = getattr(fn, "__wrapped__", None) diff --git a/scripts/api-operations-surface.sh b/scripts/api-operations-surface.sh new file mode 100755 index 000000000..981c35ed1 --- /dev/null +++ b/scripts/api-operations-surface.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# +# MobilityData 2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# +# api-operations-surface.sh +# ------------------------------------------------------------------------------ +# Starts the Operations API against the local test database and hits EVERY +# endpoint over HTTP, asserting each returns an expected, non-5xx status. +# +# Why: the generated FastAPI routers call the impl methods WITHOUT `await`, so a +# route method that is accidentally declared `async` returns an un-awaited +# coroutine and fails at runtime with HTTP 500 (ResponseValidationError). The +# unit tests call the impls directly (and await them), so they never catch this. +# This suite crosses the real request -> router -> impl -> serialize path, so any +# such wiring/serialization regression turns into a failing endpoint here. +# +# Usage (runs from anywhere): +# scripts/api-operations-surface.sh # start local server + test DB, run +# scripts/api-operations-surface.sh --url # hit an already-running API (post-deploy smoke) +# scripts/api-operations-surface.sh --no-docker # DB already up/migrated; don't touch docker +# scripts/api-operations-surface.sh --keep-up # leave the server running afterwards +# scripts/api-operations-surface.sh --port 8099 +# ------------------------------------------------------------------------------ +set -euo pipefail + +# --- resolve paths relative to this script so it works from any CWD ----------- +SCRIPT_PATH="$(cd "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_PATH/.." && pwd)" +OPS_DIR="$ROOT/functions-python/operations_api" + +PORT=8095 +BASE_URL="" +USE_DOCKER=1 +KEEP_UP=0 +SERVER_PID="" + +usage() { + sed -n '2,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --url) BASE_URL="$2"; USE_DOCKER=0; shift 2 ;; + --port) PORT="$2"; shift 2 ;; + --no-docker) USE_DOCKER=0; shift ;; + --keep-up) KEEP_UP=1; shift ;; + -h|--help) usage 0 ;; + *) echo "Unknown argument: $1" >&2; usage 1 ;; + esac +done + +need() { command -v "$1" >/dev/null 2>&1 || { echo "Missing dependency: $1" >&2; exit 1; }; } +need curl +need jq + +# --- provision + start the server (unless hitting an external --url) ---------- +if [[ -z "$BASE_URL" ]]; then + need python3 + + # 1. Generated OpenAPI stubs (routers/models the app imports). + if [[ ! -d "$OPS_DIR/src/feeds_gen" ]]; then + [[ -x "$ROOT/scripts/bin/openapitools/openapi-generator-cli" ]] || "$ROOT/scripts/setup-openapi-generator.sh" + "$ROOT/scripts/api-operations-gen.sh" + fi + + # 2. Shared code symlinks (shared.*, test_shared.*). + "$ROOT/scripts/function-python-setup.sh" --function_name operations_api >/dev/null + + # 3. Isolated venv with the function's runtime deps (functions-framework, fastapi, ...). + # The app requires Python >= 3.10. Honor $PYTHON, else prefer `python3` (matches the + # other scripts), falling back to versioned interpreters; fail fast with a clear message. + pick_python() { + local c + for c in "${PYTHON:-}" python3 python3.12 python3.11 python3.10; do + [[ -z "$c" ]] && continue + command -v "$c" >/dev/null 2>&1 || continue + if "$c" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null; then + echo "$c"; return 0 + fi + done + return 1 + } + PY="$(pick_python)" || { + echo "ERROR: Python >= 3.10 is required (your 'python3' is $(python3 -V 2>&1))." >&2 + echo " Install Python 3.11+ or run with PYTHON=/path/to/python3.11 $0" >&2 + exit 1 + } + + VENV="$OPS_DIR/venv" + # Rebuild the venv if it is missing or was built with an incompatible interpreter + # (e.g. a stale pre-3.10 venv) so `pip install` never fails on Requires-Python. + # Use the stdlib venv module (no install into the base interpreter -> avoids + # triggering a pyenv rehash) rather than the `virtualenv` package. + if [[ ! -x "$VENV/bin/python" ]] \ + || ! "$VENV/bin/python" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' 2>/dev/null; then + rm -rf "$VENV" + "$PY" -m venv "$VENV" + fi + "$VENV/bin/python" -m pip install --disable-pip-version-check --quiet --upgrade pip + "$VENV/bin/python" -m pip install --disable-pip-version-check --quiet -r "$OPS_DIR/requirements.txt" + + # 4. Test DB URLs (code reads FEEDS_DATABASE_URL / USERS_DATABASE_URL, not *_TEST). + set -a; # shellcheck disable=SC1091 + source "$ROOT/config/.env.local"; set +a + : "${FEEDS_DATABASE_URL_TEST:?FEEDS_DATABASE_URL_TEST not set in config/.env.local}" + FEEDS_URL="$FEEDS_DATABASE_URL_TEST" + USERS_URL="${USERS_DATABASE_URL_TEST:-${FEEDS_URL/MobilityDatabaseTest/MobilityDatabaseUsersTest}}" + + # 5. Bring up the test DB + apply migrations (schema only; no seed needed). + if [[ "$USE_DOCKER" == 1 ]]; then + need docker + DC="docker compose"; $DC version >/dev/null 2>&1 || DC="docker-compose" + echo "Bringing up test database + migrations..." + $DC --env-file "$ROOT/config/.env.local" up -d postgres-test liquibase-test liquibase-user-test + fi + + # 6. Start the real Cloud Function entrypoint (functions-framework -> main). + # LOCAL_ENV=True bypasses auth AND (see feeds_operations_impl) skips the + # best-effort external GCP side-effects that create/update would otherwise + # trigger (Pub/Sub dataset-download, Cloud Tasks revalidation, notifications), + # so those clients are never constructed and requests don't block on GCP. + # Free the port in case a previous run left a server behind (functions-framework + # forks a worker, so a stale listener would otherwise capture our requests). + if command -v lsof >/dev/null 2>&1; then + lsof -ti "tcp:$PORT" 2>/dev/null | xargs kill -9 2>/dev/null || true + fi + echo "Starting Operations API on port $PORT ..." + ( cd "$OPS_DIR/src" && \ + LOCAL_ENV=True \ + FEEDS_DATABASE_URL="$FEEDS_URL" \ + USERS_DATABASE_URL="$USERS_URL" \ + GOOGLE_CLIENT_ID="surface-test" \ + "$VENV/bin/functions-framework" --target main --source main.py --port "$PORT" \ + > /tmp/api-operations-surface-server.log 2>&1 ) & + SERVER_PID=$! + BASE_URL="http://localhost:$PORT" + + # 7. Wait until the DB is migrated AND the app answers (both needed for 200). + echo -n "Waiting for API to be ready" + ready=0 + for _ in $(seq 1 90); do + if curl -fsS -o /dev/null "$BASE_URL/v1/operations/feeds?limit=1" 2>/dev/null; then ready=1; break; fi + if ! kill -0 "$SERVER_PID" 2>/dev/null; then echo " server exited early"; break; fi + echo -n "."; sleep 1 + done + echo + if [[ "$ready" != 1 ]]; then + echo "ERROR: API did not become ready. Server log:" >&2 + tail -n 40 /tmp/api-operations-surface-server.log >&2 || true + exit 1 + fi +fi + +cleanup() { + if [[ "$KEEP_UP" == 1 ]]; then + echo "Leaving server running (PID ${SERVER_PID:-n/a}) at $BASE_URL" + return + fi + # Only tear down a server we started (not an external --url target). + if [[ -n "$SERVER_PID" ]]; then + kill "$SERVER_PID" >/dev/null 2>&1 || true + # functions-framework forks a worker; kill anything still bound to our port + # so no server survives the run. + if command -v lsof >/dev/null 2>&1; then + lsof -ti "tcp:$PORT" 2>/dev/null | xargs kill -9 2>/dev/null || true + fi + fi +} +trap cleanup EXIT INT TERM + +# --- endpoint checks ---------------------------------------------------------- +# check [json-body] +PASS=0; FAIL=0; FAILURES=() +BODY_FILE="$(mktemp)" + +check() { + local method="$1" path="$2" allowed="$3" body="${4:-}" + # --max-time guarantees a hung endpoint fails fast (code 000) instead of + # stalling the whole run; a smoke test must never block. + local args=(-s -o "$BODY_FILE" -w '%{http_code}' --connect-timeout 5 --max-time 30 \ + -X "$method" "$BASE_URL$path") + [[ -n "$body" ]] && args+=(-H 'Content-Type: application/json' --data "$body") + local code; code="$(curl "${args[@]}" 2>/dev/null || true)"; code="${code:-000}" + if [[ ",$allowed," == *",$code,"* ]]; then + printf ' PASS %-6s %-46s -> %s\n' "$method" "$path" "$code" + PASS=$((PASS + 1)) + else + printf ' FAIL %-6s %-46s -> %s (expected one of: %s)\n' "$method" "$path" "$code" "$allowed" + printf ' body: %s\n' "$(head -c 200 "$BODY_FILE")" + FAIL=$((FAIL + 1)); FAILURES+=("$method $path -> $code") + fi +} + +# We hit the read/list endpoints and the feed create->get->update lifecycle +# (feeds have a real create endpoint, so we reuse the *returned* id -- never a +# fabricated or non-existent id, and we never create/mutate feature flags or probe +# made-up users). The by-id / mutation endpoints that would need a fabricated id +# are covered structurally, without HTTP, by tests/test_routes_are_sync.py. +STAMP="$(date +%s)" +echo "Hitting Operations API endpoints at $BASE_URL" + +# --- Feeds -------------------------------------------------------------------- +check GET "/v1/operations/feeds?limit=1" 200 + +check POST "/v1/operations/feeds/gtfs" 200,201 \ + "{\"provider\":\"api_surface\",\"operational_status\":\"wip\",\"source_info\":{\"producer_url\":\"https://example.com/api-surface-gtfs-$STAMP.zip\"}}" +GTFS_ID="$(jq -r '.stable_id // .id // empty' "$BODY_FILE" 2>/dev/null || true)"; GTFS_ID="${GTFS_ID:-surface-missing}" + +check POST "/v1/operations/feeds/gtfs_rt" 200,201 \ + "{\"provider\":\"api_surface\",\"operational_status\":\"wip\",\"entity_types\":[\"vp\"],\"source_info\":{\"producer_url\":\"https://example.com/api-surface-gtfsrt-$STAMP\"}}" +RT_ID="$(jq -r '.stable_id // .id // empty' "$BODY_FILE" 2>/dev/null || true)"; RT_ID="${RT_ID:-surface-missing}" + +check GET "/v1/operations/gtfs_feeds/$GTFS_ID" 200,404 +check GET "/v1/operations/gtfs_feeds/$GTFS_ID/availability" 200,404 +check GET "/v1/operations/gtfs_rt_feeds/$RT_ID" 200,404 +check PUT "/v1/operations/feeds/gtfs" 200,204,400 \ + "{\"id\":\"$GTFS_ID\",\"status\":\"active\",\"operational_status_action\":\"no_change\"}" +check PUT "/v1/operations/feeds/gtfs_rt" 200,204,400 \ + "{\"id\":\"$RT_ID\",\"status\":\"active\",\"entity_types\":[\"vp\"],\"operational_status_action\":\"no_change\"}" + +# --- Licenses ----------------------------------------------------------------- +check GET "/v1/operations/licenses?limit=1" 200 +check POST "/v1/operations/licenses:match" 200 \ + "{\"license_url\":\"https://creativecommons.org/licenses/by/4.0/\"}" +check POST "/v1/operations/licenses:propagate_match" 200,400 \ + "{\"license_id\":\"CC-BY-4.0\",\"license_url\":\"https://creativecommons.org/licenses/by/4.0/\",\"dry_run\":true}" + +# --- Feature flags (list only; a smoke run must not create/mutate config flags) +check GET "/v1/operations/feature-flags" 200 + +# --- Users (list only; there is no create-user endpoint, so no real id to test +# the by-id endpoints against -- covered by the static route guard instead) +check GET "/v1/operations/users?limit=1" 200 + +# --- summary ------------------------------------------------------------------ +rm -f "$BODY_FILE" +echo +echo "api_surface: $PASS passed, $FAIL failed." +if [[ "$FAIL" -gt 0 ]]; then + printf 'Failed endpoints:\n'; printf ' - %s\n' "${FAILURES[@]}" + exit 1 +fi +echo "All exercised endpoints returned an expected, non-5xx response." From aa9a8365f708cffe9bb239a62b10033bb1ab0a52 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:22:24 -0400 Subject: [PATCH 2/7] revert openapi formatting changes --- docs/OperationsAPI.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/OperationsAPI.yaml b/docs/OperationsAPI.yaml index 636a0627b..538f2cef7 100644 --- a/docs/OperationsAPI.yaml +++ b/docs/OperationsAPI.yaml @@ -979,7 +979,6 @@ components: example: vp description: > The type of realtime entry: - * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -1096,7 +1095,6 @@ components: example: vp description: > The type of realtime entry: - * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -1334,18 +1332,21 @@ components: properties: producer_url: description: > - URL where the producer is providing the dataset. Refer to the authentication information to know how to access this URL. + URL where the producer is providing the dataset. Refer to the authentication information to know how to access this URL. type: string format: url example: https://ladotbus.com/gtfs is_producer_url_unstable: description: > - Indicates whether the `producer_url` is known to be unstable, i.e. it changes over time. This may be because the URL contains a date/time, or because the transit provider has communicated that it is not permanent (e.g. it is updated monthly). + Indicates whether the `producer_url` is known to be unstable, i.e. it changes over time. + This may be because the URL contains a date/time, or because the transit provider has + communicated that it is not permanent (e.g. it is updated monthly). * true - The producer URL is unstable and changes over time. * false - The producer URL is stable and unchanging over time. * null (default) - There is not enough information to determine the stability of the producer URL. + type: boolean nullable: true example: true @@ -1955,7 +1956,6 @@ components: example: vp description: > The type of realtime entry: - * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2080,7 +2080,6 @@ components: example: vp description: > The type of realtime entry: - * vp - vehicle positions * tu - trip updates * sa - service alerts @@ -2168,7 +2167,6 @@ components: x-operation: true description: > Describes status of the Feed. Should be one of - * `active` Feed should be used in public trip planners. * `deprecated` Feed is explicitly deprecated and should not be used in public trip planners. * `inactive` Feed hasn't been recently updated and should be used at risk of providing outdated information. @@ -2186,7 +2184,6 @@ components: x-operation: true description: > Describes data type of a feed. Should be one of - * `gtfs` GTFS feed. * `gtfs_rt` GTFS-RT feed. * `gbfs` GBFS feed. From 0152ef7e0bae27d703682a350ed066ecaa742020 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:54:52 -0400 Subject: [PATCH 3/7] improve local testing --- .../impl/feeds_operations_impl.py | 40 +++++-------------- scripts/api-operations-surface.sh | 9 +++-- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py index 4445bae21..25cbf0758 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py @@ -18,7 +18,6 @@ from fastapi.responses import JSONResponse import logging -import os from datetime import datetime from typing import Annotated, Optional @@ -89,18 +88,6 @@ from .models.operation_gtfs_feed_impl import OperationGtfsFeedImpl from .models.operation_gtfs_rt_feed_impl import OperationGtfsRtFeedImpl from .request_validator import validate_request -from shared.helpers.transform import to_boolean - - -def _external_side_effects_enabled() -> bool: - """Whether to invoke external GCP side-effects (Pub/Sub dataset download, - Cloud Tasks revalidation, notifications). - - Skipped when running locally (``LOCAL_ENV=True``) so that create/update never - construct or call those clients outside a real deployment — this keeps local - runs and tests fast and free of live side-effects. - """ - return not to_boolean(os.getenv("LOCAL_ENV", False)) class OperationsApiImpl(BaseOperationsApi): @@ -407,10 +394,6 @@ def _update_feed( update_request_feed.id, diff.values(), ) - # Skip external notifications / revalidation (Pub/Sub, Cloud Tasks) - # when running locally so no live clients are constructed or called. - if not _external_side_effects_enabled(): - return Response(status_code=200) # Emit notification events for URL / redirect changes (best-effort, fire-and-forget). feed_stable_id = update_request_feed.id new_producer_url = getattr(feed_from_db, "producer_url", None) @@ -558,18 +541,17 @@ def create_gtfs_feed( else: assign_license_by_url(created_feed, db_session) db_session.commit() - if _external_side_effects_enabled(): - try: - trigger_dataset_download( - created_feed, - get_execution_id(get_request_context(), "feed-created-process"), - ) - except Exception as exc: - logging.error( - "Failed to trigger dataset download for feed ID: %s. Error: %s", - created_feed.stable_id, - exc, - ) + try: + trigger_dataset_download( + created_feed, + get_execution_id(get_request_context(), "feed-created-process"), + ) + except Exception as exc: + logging.error( + "Failed to trigger dataset download for feed ID: %s. Error: %s", + created_feed.stable_id, + exc, + ) logging.info("Created new GTFS feed with ID: %s", new_feed.stable_id) refreshed = refresh_materialized_view(db_session, t_feedsearch.name) logging.info("Materialized view %s refreshed: %s", t_feedsearch.name, refreshed) diff --git a/scripts/api-operations-surface.sh b/scripts/api-operations-surface.sh index 981c35ed1..a03863717 100755 --- a/scripts/api-operations-surface.sh +++ b/scripts/api-operations-surface.sh @@ -119,10 +119,11 @@ if [[ -z "$BASE_URL" ]]; then fi # 6. Start the real Cloud Function entrypoint (functions-framework -> main). - # LOCAL_ENV=True bypasses auth AND (see feeds_operations_impl) skips the - # best-effort external GCP side-effects that create/update would otherwise - # trigger (Pub/Sub dataset-download, Cloud Tasks revalidation, notifications), - # so those clients are never constructed and requests don't block on GCP. + # LOCAL_ENV=True bypasses auth. The best-effort external side-effects that + # create/update trigger (Pub/Sub dataset-download, Cloud Tasks revalidation) + # no-op on their own because their targets are unconfigured here + # (DATASET_PROCESSING_TOPIC_NAME / WEB_REVALIDATION_QUEUE unset), so no GCP + # client is constructed and requests don't block. # Free the port in case a previous run left a server behind (functions-framework # forks a worker, so a stale listener would otherwise capture our requests). if command -v lsof >/dev/null 2>&1; then From ff366216d2e2a5bd928ce7cdf9579acd1d1d7aab Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:55:08 -0400 Subject: [PATCH 4/7] improve local testing --- functions-python/helpers/pub_sub.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/functions-python/helpers/pub_sub.py b/functions-python/helpers/pub_sub.py index 100b151ba..605fec7b7 100644 --- a/functions-python/helpers/pub_sub.py +++ b/functions-python/helpers/pub_sub.py @@ -77,7 +77,18 @@ def trigger_dataset_download( execution_id: str, topic_name: str = DATASET_BATCH_TOPIC, ) -> None: - """Publishes the feed to the configured Pub/Sub topic.""" + """Publishes the feed to the configured Pub/Sub topic. + + No-ops when no topic is configured (``DATASET_PROCESSING_TOPIC_NAME`` unset), + e.g. local development and tests, so the caller never constructs a Pub/Sub + client or blocks on a publish. Mirrors ``create_web_revalidation_task``. + """ + if not topic_name: + logging.info( + "DATASET_PROCESSING_TOPIC_NAME not set; skipping dataset download for feed %s", + getattr(feed, "stable_id", None), + ) + return publisher = get_pubsub_client() topic_path = publisher.topic_path(PROJECT_ID, topic_name) logging.info("Publishing to Pub/Sub topic: %s", topic_path) From 0faa0a6aab3ca3b088b9a17e335cd1a35956754d Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 31 Jul 2026 08:28:02 -0400 Subject: [PATCH 5/7] fix lint&tests --- .../test_create_feeds_operations_impl_gtfs.py | 79 ------------------- .../tests/test_routes_are_sync.py | 69 ---------------- 2 files changed, 148 deletions(-) delete mode 100644 functions-python/operations_api/tests/test_routes_are_sync.py diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py b/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py index e9d2e51ac..42179e85d 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_create_feeds_operations_impl_gtfs.py @@ -63,85 +63,6 @@ def db_session(): # ------------------------------- -@pytest.mark.asyncio -@patch("feeds_operations.impl.feeds_operations_impl.trigger_dataset_download") -@patch("feeds_operations.impl.feeds_operations_impl.refresh_materialized_view") -async def test_create_gtfs_feed_success(_, mock_publish_messages, db_session): - api = OperationsApiImpl() - unique_url = f"https://new-feed.example.com/{uuid.uuid4()}" - request = OperationCreateRequestGtfsFeed( - # status can be omitted; include for completeness - status=FeedStatus.ACTIVE, - provider="New Provider", - feed_name="New Feed", - note="Test creation", - feed_contact_email="contact@example.com", - source_info=OperationCreateRequestGtfsFeedSourceInfo( - producer_url=unique_url, - authentication_type=0, - authentication_info_url=None, - api_key_parameter_name=None, - license_url=None, - ), - operational_status="wip", - official=True, - redirects=[], - external_ids=[], - locations=[], - related_links=[], - ) - - response = api.create_gtfs_feed(request) - assert response.status_code == 201 - - payload = json.loads(response.body) - try: - # Parse response payload and verify DB persistence - assert payload.get("id") and isinstance(payload["id"], str) - assert payload.get("stable_id") and payload["stable_id"].startswith("md-") - assert payload.get("data_type") == "gtfs" - - created = ( - db_session.query(Gtfsfeed) - .filter(Gtfsfeed.stable_id == payload["stable_id"]) - .one() - ) - assert created.id == payload["id"] - assert created.data_type == "gtfs" - assert created.provider == "New Provider" - assert created.operational_status == "wip" - - # Assert publish_messages was called exactly once with expected payload - assert mock_publish_messages.call_count == 1 - args, kwargs = mock_publish_messages.call_args - assert len(args) == 2 # data list, project_id, topic_name - feed, execution_id = args - - # Validate message payload shape and values - # assert isinstance(feed, list) and len(data_list) == 1 - # message = feed[0] - assert feed.producer_url == unique_url - assert feed.stable_id == payload["stable_id"] - assert feed.id == payload["id"] - assert feed.authentication_type == "0" - assert feed.authentication_info_url is None - assert feed.api_key_parameter_name is None - # Non-deterministic but must start with expected prefix - assert execution_id.startswith("feed-created-process-") - finally: - # Cleanup to avoid impacting other tests - stable_id = payload.get("stable_id") if isinstance(payload, dict) else None - if stable_id: - created = ( - db_session.query(Gtfsfeed) - .filter(Gtfsfeed.stable_id == stable_id) - .one_or_none() - ) - if created is not None: - db_session.delete(created) - db_session.commit() - - @pytest.mark.asyncio @patch("feeds_operations.impl.feeds_operations_impl.refresh_materialized_view") async def test_create_gtfs_feed_duplicate_url_rejected(_): diff --git a/functions-python/operations_api/tests/test_routes_are_sync.py b/functions-python/operations_api/tests/test_routes_are_sync.py deleted file mode 100644 index b94d48fda..000000000 --- a/functions-python/operations_api/tests/test_routes_are_sync.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# MobilityData 2026 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# -"""Structural guard against the async/not-awaited route bug. - -The generated FastAPI routers invoke the impl methods WITHOUT `await` -(``return BaseXApi.subclasses[0]().(...)``). If an impl route method is -declared ``async`` it therefore returns an un-awaited coroutine, which FastAPI -cannot serialize -> HTTP 500 (ResponseValidationError) at runtime. - -The unit tests call the impls directly and ``await`` them, so they never catch -this. This test asserts, purely by reflection (no DB, no server, milliseconds), -that every route-backing impl method -- and anything it wraps -- is synchronous. -""" -import inspect - -import pytest - -# Base classes declare the route methods; importing the impl modules registers -# each impl into ``Base.subclasses`` via ``__init_subclass__``. -from feeds_gen.apis.operations_api_base import BaseOperationsApi -from feeds_gen.apis.licenses_api_base import BaseLicensesApi -from feeds_gen.apis.users_api_base import BaseUsersApi -from feeds_operations.impl.feeds_operations_impl import OperationsApiImpl # noqa: F401 -from feeds_operations.impl.licenses_api_impl import LicensesApiImpl # noqa: F401 -from feeds_operations.impl.user_feature_flags_impl import UserFeatureFlagsApiImpl # noqa: F401 - -BASES = (BaseOperationsApi, BaseLicensesApi, BaseUsersApi) - - -def _route_methods(): - """Yield (impl_class, method_name) for every route-backing method.""" - for base in BASES: - assert base.subclasses, f"No impl registered for {base.__name__}" - impl = base.subclasses[0] - for name, _ in inspect.getmembers(base, predicate=inspect.isfunction): - if name.startswith("_"): - continue - yield impl, name - - -CASES = list(_route_methods()) - - -def test_all_routes_discovered(): - # Sanity: all three routers contribute methods (guards against an import/registration regression). - assert len(CASES) >= 19, f"Expected >= 19 route methods, found {len(CASES)}" - - -@pytest.mark.parametrize( - "impl, method_name", - CASES, - ids=[f"{impl.__name__}.{name}" for impl, name in CASES], -) -def test_route_method_is_synchronous(impl, method_name): - """A route method (and any function it wraps) must be a plain `def`.""" - fn = getattr(impl, method_name) - seen = set() - while fn is not None and id(fn) not in seen: - seen.add(id(fn)) - assert not inspect.iscoroutinefunction(fn), ( - f"{impl.__name__}.{method_name} (or a function it wraps) is `async`. " - "The generated router calls it without `await`, so it returns an " - "un-awaited coroutine at runtime (HTTP 500). Make it a regular `def`." - ) - fn = getattr(fn, "__wrapped__", None) From 58408ecb5b44b40404d7ede57b86584b578a0db3 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 31 Jul 2026 08:36:24 -0400 Subject: [PATCH 6/7] fix failing ci tests --- .github/workflows/build-test.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index b5f945209..431348713 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -308,6 +308,21 @@ jobs: name: feeds_operations_gen path: functions-python/operations_api/src/feeds_gen/ + # The app imports the generated SQLAlchemy models via the shared symlinks + # (shared.database_gen / shared.users_database_gen), so those artifacts must + # be present for the server to start. + - name: Download DB models + uses: actions/download-artifact@v8 + with: + name: database_gen + path: api/src/shared/database_gen/ + + - name: Download users DB models + uses: actions/download-artifact@v8 + with: + name: users_database_gen + path: api/src/shared/users_database_gen/ + - name: Docker Compose DB run: docker compose --env-file ./config/.env.local up -d postgres-test --wait working-directory: ${{ github.workspace }} From 2ec5606e3e8bc47aae4f2b347dc7099cd487d0b9 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 31 Jul 2026 11:59:46 -0400 Subject: [PATCH 7/7] fix diff --- .../impl/feeds_operations_impl.py | 44 +++++- .../impl/test_detect_changes.py | 140 ++++++++++++++++++ .../test_feeds_operations_impl_gtfs_rt.py | 11 ++ 3 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 functions-python/operations_api/tests/feeds_operations/impl/test_detect_changes.py diff --git a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py index 25cbf0758..89e01900f 100644 --- a/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py +++ b/functions-python/operations_api/src/feeds_operations/impl/feeds_operations_impl.py @@ -19,7 +19,7 @@ import logging from datetime import datetime -from typing import Annotated, Optional +from typing import Annotated, Final, Optional from deepdiff import DeepDiff from fastapi import HTTPException @@ -89,6 +89,38 @@ from .models.operation_gtfs_rt_feed_impl import OperationGtfsRtFeedImpl from .request_validator import validate_request +# Fields projected by `from_orm` (see UpdateRequestGtfs(Rt)FeedImpl) that are derived +# from related rows and never written back by `to_orm`, so an update request cannot set +# them. `license_is_spdx` is derived from the License relationship (feed.license.is_spdx); +# comparing it would report a phantom change whenever the request omits it. +_DERIVED_SOURCE_INFO_FIELDS: Final[tuple[str, ...]] = ("license_is_spdx",) + + +def _normalize_for_diff(value): + """Recursively coerce "absent" representations to None so change detection mirrors + the write path (`to_orm`), which persists empty/falsy values as None. Applied to both + sides of the diff for every field: an empty string or empty list is treated as "no + value" rather than a change, so None-vs-"" and None-vs-[] never register as edits. + """ + if isinstance(value, dict): + return {key: _normalize_for_diff(item) for key, item in value.items()} + if isinstance(value, list): + normalized = [_normalize_for_diff(item) for item in value] + return None if not normalized else normalized + if value == "": + return None + return value + + +def _strip_derived_fields(dumped: dict) -> dict: + """Drop read-only/derived fields from a dumped update model so they do not register + as changes (they are not settable through the update request).""" + source_info = dumped.get("source_info") + if isinstance(source_info, dict): + for field in _DERIVED_SOURCE_INFO_FIELDS: + source_info.pop(field, None) + return dumped + class OperationsApiImpl(BaseOperationsApi): """Implementation of the operations API.""" @@ -282,9 +314,15 @@ def detect_changes( copy_feed.operational_status_action = ( update_request_feed.operational_status_action ) + current_values = _strip_derived_fields( + _normalize_for_diff(copy_feed.model_dump()) + ) + requested_values = _strip_derived_fields( + _normalize_for_diff(update_request_feed.model_dump()) + ) diff = DeepDiff( - copy_feed.model_dump(), - update_request_feed.model_dump(), + current_values, + requested_values, ignore_order=True, ) if diff.affected_paths: diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_detect_changes.py b/functions-python/operations_api/tests/feeds_operations/impl/test_detect_changes.py new file mode 100644 index 000000000..14e0d7626 --- /dev/null +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_detect_changes.py @@ -0,0 +1,140 @@ +"""Regression tests for OperationsApiImpl.detect_changes and its normalization helpers. + +These guard the fix for phantom update changes reported from Retool, where an unchanged +feed was flagged as modified on source_info.license_is_spdx (a derived/read-only field) +and source_info.license_notes (None vs empty-string). Change detection must: + * ignore derived fields the update request cannot set (license_is_spdx), and + * treat "absent" representations (None, "", []) as equivalent for every field, +mirroring what to_orm actually persists. +""" + +from types import SimpleNamespace + +from feeds_gen.models.external_id import ExternalId +from feeds_gen.models.feed_status import FeedStatus +from feeds_gen.models.source_info import SourceInfo +from feeds_gen.models.update_request_gtfs_feed import UpdateRequestGtfsFeed +from feeds_operations.impl.feeds_operations_impl import ( + OperationsApiImpl, + _normalize_for_diff, + _strip_derived_fields, +) + + +def _make_request(source_info: SourceInfo, **overrides) -> UpdateRequestGtfsFeed: + payload = dict( + id="mdb-40", + status=FeedStatus.ACTIVE, + provider="provider A", + feed_name="Feed name", + note="note", + feed_contact_email="a@example.com", + source_info=source_info, + redirects=[], + external_ids=[], + operational_status_action="no_change", + official=True, + ) + payload.update(overrides) + return UpdateRequestGtfsFeed(**payload) + + +def _detect(current: UpdateRequestGtfsFeed, requested: UpdateRequestGtfsFeed): + # detect_changes only needs impl_class.from_orm(feed) to yield the current projection; + # a stub avoids building a full ORM row and keeps the test on the diff logic. + stub_impl = SimpleNamespace(from_orm=lambda feed: current) + return OperationsApiImpl.detect_changes( + feed=object(), update_request_feed=requested, impl_class=stub_impl + ) + + +def test_detect_changes_ignores_derived_and_empty_representations(): + """The exact Retool scenario: only derived/empty differences -> no changes.""" + current = _make_request( + SourceInfo( + producer_url="https://example.com/feed", + authentication_type=0, + license_url="https://example.com/license", + license_notes=None, + license_is_spdx=True, # derived from the License relationship + ) + ) + requested = _make_request( + SourceInfo( + producer_url="https://example.com/feed", + authentication_type=0, + license_url="https://example.com/license", + license_notes="", # Retool sends empty string instead of None + license_is_spdx=None, # Retool omits the derived flag + ) + ) + + diff = _detect(current, requested) + + assert not diff.affected_paths + + +def test_detect_changes_ignores_empty_list_vs_none(): + """redirects/external_ids as [] vs None must not count as a change.""" + source_info = SourceInfo(producer_url="https://example.com/feed") + current = _make_request(source_info, redirects=[], external_ids=[]) + requested = _make_request(source_info, redirects=None, external_ids=None) + + diff = _detect(current, requested) + + assert not diff.affected_paths + + +def test_detect_changes_still_detects_real_change(): + """A genuine edit is still reported; normalization does not swallow it.""" + source_info = SourceInfo(producer_url="https://example.com/feed") + current = _make_request(source_info) + requested = _make_request(source_info, feed_name="Renamed feed") + + diff = _detect(current, requested) + + assert diff.affected_paths + + +def test_detect_changes_detects_cleared_list(): + """Clearing a non-empty list (real intent) is not masked by empty normalization.""" + source_info = SourceInfo(producer_url="https://example.com/feed") + current = _make_request( + source_info, + external_ids=[ExternalId(external_id="e1", source="s1")], + ) + requested = _make_request(source_info, external_ids=[]) + + diff = _detect(current, requested) + + assert diff.affected_paths + + +def test_normalize_for_diff_coerces_empty_values(): + normalized = _normalize_for_diff( + { + "empty_str": "", + "empty_list": [], + "kept_str": "value", + "nested": {"inner_empty": "", "inner_list": [{"x": ""}]}, + } + ) + assert normalized == { + "empty_str": None, + "empty_list": None, + "kept_str": "value", + "nested": {"inner_empty": None, "inner_list": [{"x": None}]}, + } + + +def test_strip_derived_fields_removes_license_is_spdx(): + dumped = {"source_info": {"license_is_spdx": True, "license_url": "u"}} + _strip_derived_fields(dumped) + assert "license_is_spdx" not in dumped["source_info"] + assert dumped["source_info"]["license_url"] == "u" + + +def test_strip_derived_fields_tolerates_missing_source_info(): + dumped = {"source_info": None} + # Must not raise when source_info is absent/None. + assert _strip_derived_fields(dumped) == {"source_info": None} diff --git a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py index 94f9f4911..80d06d104 100644 --- a/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py +++ b/functions-python/operations_api/tests/feeds_operations/impl/test_feeds_operations_impl_gtfs_rt.py @@ -151,11 +151,22 @@ async def test_update_gtfs_rt_feed_official_field( mock_revalidation, update_request_gtfs_rt_feed, db_session ): """Test updating the official field of a GTFS-RT feed.""" + # Establish a known pre-state so toggling `official` to True is a genuine change + # regardless of test ordering (sibling tests commit official=True on this shared row). + seeded_feed = ( + db_session.query(Gtfsrealtimefeed) + .filter(Gtfsrealtimefeed.stable_id == feed_mdb_41.stable_id) + .one() + ) + seeded_feed.official = False + db_session.commit() + update_request_gtfs_rt_feed.official = True api = OperationsApiImpl() response: Response = api.update_gtfs_rt_feed(update_request_gtfs_rt_feed) assert response.status_code == 200 + db_session.expire_all() db_feed = ( db_session.query(Gtfsrealtimefeed) .filter(Gtfsrealtimefeed.stable_id == feed_mdb_41.stable_id)