From 78b03aa12b93ba3d5b108a1155d53f54220780ab Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 19 Jun 2026 11:01:43 -0400 Subject: [PATCH 01/17] gateway: federation export API, Redis events, sync API key --- gateway/.envs/example/django.env | 11 +++ gateway/config/api_router.py | 2 + gateway/config/settings/base.py | 19 +++- gateway/sds_gateway/api_methods/apps.py | 1 + .../sds_gateway/api_methods/authentication.py | 4 +- .../api_methods/federation/__init__.py | 0 .../api_methods/federation/events.py | 39 ++++++++ .../api_methods/federation/export_contract.py | 26 +++++ .../api_methods/federation/signals.py | 71 ++++++++++++++ .../helpers/compile_federated_data.py | 78 +++++++++++++++ gateway/sds_gateway/api_methods/models.py | 1 + .../sds_gateway/api_methods/permissions.py | 31 ++++++ .../serializers/capture_serializers.py | 53 ++++++++++ .../serializers/dataset_serializers.py | 40 ++++++++ .../api_methods/tests/test_authenticate.py | 2 +- .../tests/test_federation_export.py | 98 +++++++++++++++++++ .../tests/test_federation_export_contract.py | 94 ++++++++++++++++++ .../api_methods/views/capture_endpoints.py | 2 - .../api_methods/views/dataset_endpoints.py | 4 +- .../api_methods/views/federation_endpoints.py | 87 ++++++++++++++++ .../api_methods/views/file_endpoints.py | 3 - .../create_federation_sync_api_key.py | 46 +++++++++ ...alter_userapikey_source_federation_sync.py | 38 +++++++ .../users/migrations/max_migration.txt | 2 +- gateway/sds_gateway/users/models.py | 1 + 25 files changed, 740 insertions(+), 13 deletions(-) create mode 100644 gateway/sds_gateway/api_methods/federation/__init__.py create mode 100644 gateway/sds_gateway/api_methods/federation/events.py create mode 100644 gateway/sds_gateway/api_methods/federation/export_contract.py create mode 100644 gateway/sds_gateway/api_methods/federation/signals.py create mode 100644 gateway/sds_gateway/api_methods/helpers/compile_federated_data.py create mode 100644 gateway/sds_gateway/api_methods/permissions.py create mode 100644 gateway/sds_gateway/api_methods/tests/test_federation_export.py create mode 100644 gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py create mode 100644 gateway/sds_gateway/api_methods/views/federation_endpoints.py create mode 100644 gateway/sds_gateway/users/management/commands/create_federation_sync_api_key.py create mode 100644 gateway/sds_gateway/users/migrations/0012_alter_userapikey_source_federation_sync.py diff --git a/gateway/.envs/example/django.env b/gateway/.envs/example/django.env index 1287a26fa..77e2990f8 100644 --- a/gateway/.envs/example/django.env +++ b/gateway/.envs/example/django.env @@ -8,6 +8,17 @@ HOSTNAME=localhost:8000 API_VERSION=v1 API_KEY= +# FEDERATION +# ------------------------------------------------------------------------------ +# the below environment variables are used for site federation with peers. +# By default, federation events are disabled. +# If FEDERATION_EVENTS_ENABLED is TRUE, deployment will need to include +# federation sync service configuration. +FEDERATION_SITE_NAME= +FEDERATION_EVENTS_ENABLED=false +FEDERATION_EVENTS_CHANNEL=federation:events +FEDERATION_SYNC_USER_EMAIL= + # AUTH0 # ------------------------------------------------------------------------------ # Set these from your Auth0 application settings diff --git a/gateway/config/api_router.py b/gateway/config/api_router.py index cc9a80f44..976ff5023 100644 --- a/gateway/config/api_router.py +++ b/gateway/config/api_router.py @@ -7,6 +7,7 @@ from sds_gateway.api_methods.views.dataset_endpoints import DatasetViewSet from sds_gateway.api_methods.views.file_endpoints import FileViewSet from sds_gateway.api_methods.views.file_endpoints import check_contents_exist +from sds_gateway.api_methods.views.federation_endpoints import FederationViewSet from sds_gateway.users.api.views import UserViewSet from sds_gateway.visualizations.api_views import VisualizationViewSet @@ -17,6 +18,7 @@ router.register(r"assets/files", FileViewSet, basename="files") router.register(r"assets/captures", CaptureViewSet, basename="captures") router.register(r"assets/datasets", DatasetViewSet, basename="datasets") +router.register(r"federation", FederationViewSet, basename="federation") if settings.VISUALIZATIONS_ENABLED: router.register(r"visualizations", VisualizationViewSet, basename="visualizations") diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index 87fb85ab4..8c483a22d 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -610,7 +610,10 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: "rest_framework.authentication.SessionAuthentication", "sds_gateway.api_methods.authentication.APIKeyAuthentication", ), - "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), + "DEFAULT_PERMISSION_CLASSES": ( + "rest_framework.permissions.IsAuthenticated", + "sds_gateway.api_methods.permissions.DisallowFederationSyncKey", + ), "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", "DEFAULT_THROTTLE_RATES": { "vis_stream": VIS_STREAM_THROTTLE_RATE, @@ -709,6 +712,20 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: SDS_PROGRAMMATIC_SITE_NAME: str = env.str("SDS_PROGRAMMATIC_SITE_NAME", default="sds") SDS_SITE_FQDN: str = env.str("SDS_SITE_FQDN", default="localhost") +FEDERATION_SITE_NAME: str = env.str( + "FEDERATION_SITE_NAME", + default="", +) +FEDERATION_EVENTS_ENABLED: bool = env.bool("FEDERATION_EVENTS_ENABLED", default=False) +FEDERATION_EVENTS_CHANNEL: str = env.str( + "FEDERATION_EVENTS_CHANNEL", + default="federation:events", +) +FEDERATION_SYNC_USER_EMAIL: str = env.str( + "FEDERATION_SYNC_USER_EMAIL", + default="federation-sync@internal.local", +) + # ADMIN_CONSOLE_ENV is used to visually distinguish between different environments # (production, staging, local) in the admin console and error emails. It does not affect # any functionality and it is meant to prevent changes in production meant for testing diff --git a/gateway/sds_gateway/api_methods/apps.py b/gateway/sds_gateway/api_methods/apps.py index 73073e45a..2c38194ff 100644 --- a/gateway/sds_gateway/api_methods/apps.py +++ b/gateway/sds_gateway/api_methods/apps.py @@ -12,6 +12,7 @@ class ApiMethodsConfig(AppConfig): # pattern to import application modules here in ready() # ruff: noqa: PLC0415 def ready(self) -> None: + import sds_gateway.api_methods.federation.signals # noqa: F401 import sds_gateway.api_methods.schema # noqa: F401 silence_unwanted_logs() diff --git a/gateway/sds_gateway/api_methods/authentication.py b/gateway/sds_gateway/api_methods/authentication.py index 2a210478f..60e831970 100644 --- a/gateway/sds_gateway/api_methods/authentication.py +++ b/gateway/sds_gateway/api_methods/authentication.py @@ -13,7 +13,7 @@ class APIKeyAuthentication(BaseAuthentication): keyword = "Api-Key" - def authenticate(self, request) -> tuple[User, bool]: + def authenticate(self, request) -> tuple[User, UserAPIKey]: """Authenticates the user with their API key. Args: request: Contains the API key in the Authorization header. @@ -47,7 +47,7 @@ def authenticate(self, request) -> tuple[User, bool]: raise AuthenticationFailed(msg) from err user = api_key_obj.user - return (user, True) + return (user, api_key_obj) def authenticate_header(self, request) -> str: return self.keyword diff --git a/gateway/sds_gateway/api_methods/federation/__init__.py b/gateway/sds_gateway/api_methods/federation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gateway/sds_gateway/api_methods/federation/events.py b/gateway/sds_gateway/api_methods/federation/events.py new file mode 100644 index 000000000..f1d8ff897 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/events.py @@ -0,0 +1,39 @@ +"""Publish federation change notifications to Redis.""" + +from __future__ import annotations + +import json +from datetime import UTC +from datetime import datetime +from typing import Any +from uuid import UUID + +from django.conf import settings +from loguru import logger as log + +from sds_gateway.api_methods.models import ItemType +from sds_gateway.api_methods.tasks import get_redis_client + +FederationEventType = str # created | updated | deleted + + +def publish_federation_event( + *, + event_type: FederationEventType, + item_type: ItemType, + uuid: UUID, + timestamp: datetime | None = None, +) -> None: + """Notify the local federation sync service via Redis pub/sub.""" + channel = getattr(settings, "FEDERATION_EVENTS_CHANNEL", "federation:events") + payload: dict[str, Any] = { + "event_type": event_type, + "item_type": item_type.value, + "uuid": str(uuid), + "timestamp": (timestamp or datetime.now(UTC)).isoformat(), + } + try: + client = get_redis_client() + client.publish(channel, json.dumps(payload)) + except Exception as err: # noqa: BLE001 + log.warning("Failed to publish federation event: {}", err) diff --git a/gateway/sds_gateway/api_methods/federation/export_contract.py b/gateway/sds_gateway/api_methods/federation/export_contract.py new file mode 100644 index 000000000..7d9931f67 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/export_contract.py @@ -0,0 +1,26 @@ +"""Helpers to keep gateway export serializers aligned with sync Pydantic models.""" + +from __future__ import annotations + +from typing import Any + +from rest_framework.serializers import BaseSerializer + + +def serializer_output_field_names(serializer: BaseSerializer[Any]) -> set[str]: + return set(serializer.fields.keys()) + + +def assert_field_names_match( + serializer: BaseSerializer[Any], + pydantic_model: type, + *, + label: str, +) -> None: + expected = set(pydantic_model.model_fields.keys()) + actual = serializer_output_field_names(serializer) + if expected != actual: + missing = expected - actual + extra = actual - expected + msg = f"{label} field mismatch: missing={sorted(missing)!r} extra={sorted(extra)!r}" + raise AssertionError(msg) diff --git a/gateway/sds_gateway/api_methods/federation/signals.py b/gateway/sds_gateway/api_methods/federation/signals.py new file mode 100644 index 000000000..c65619971 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/signals.py @@ -0,0 +1,71 @@ +"""Django signals that publish federation Redis events.""" + +from __future__ import annotations + +from django.conf import settings +from django.db.models.signals import post_save +from django.dispatch import receiver +from loguru import logger as log + +from sds_gateway.api_methods.federation.events import publish_federation_event +from sds_gateway.api_methods.helpers.compile_federated_data import ( + is_federation_exportable_capture, +) +from sds_gateway.api_methods.helpers.compile_federated_data import ( + is_federation_exportable_dataset, +) +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import ItemType + + +def _event_type(*, created: bool, exportable: bool) -> str: + if not exportable: + return "deleted" + return "created" if created else "updated" + + +@receiver(post_save, sender=Dataset) +def federation_dataset_changed( + sender: type[Dataset], + instance: Dataset, + created: bool, + **kwargs, +) -> None: + if not getattr(settings, "FEDERATION_EVENTS_ENABLED", True): + log.debug( + "FEDERATION_EVENTS_ENABLED is False, " + "skipping federation dataset changed signal", + ) + return + + exportable = is_federation_exportable_dataset(instance) + publish_federation_event( + event_type=_event_type(created=created, exportable=exportable), + item_type=ItemType.DATASET, + uuid=instance.uuid, + timestamp=instance.updated_at, + ) + + +@receiver(post_save, sender=Capture) +def federation_capture_changed( + sender: type[Capture], + instance: Capture, + created: bool, + **kwargs, +) -> None: + if not getattr(settings, "FEDERATION_EVENTS_ENABLED", True): + log.debug( + "FEDERATION_EVENTS_ENABLED is False, " + "skipping federation capture changed signal", + ) + return + + exportable = is_federation_exportable_capture(instance) + publish_federation_event( + event_type=_event_type(created=created, exportable=exportable), + item_type=ItemType.CAPTURE, + uuid=instance.uuid, + timestamp=instance.updated_at, + ) diff --git a/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py b/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py new file mode 100644 index 000000000..48db49377 --- /dev/null +++ b/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py @@ -0,0 +1,78 @@ +"""Build federation export payloads (RFC fed-* index shape).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any + +from django.conf import settings + +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.serializers.dataset_serializers import ( + get_dataset_serializer, +) +from sds_gateway.api_methods.serializers.dataset_serializers import ( + DatasetFederationSerializer, +) +from sds_gateway.api_methods.serializers.capture_serializers import ( + CaptureFederationSerializer, +) + +if TYPE_CHECKING: + from django.db.models import QuerySet + + +def federation_site_name() -> str: + return getattr(settings, "FEDERATION_SITE_NAME", settings.SDS_PROGRAMMATIC_SITE_NAME) + + +def public_datasets_queryset() -> QuerySet[Dataset]: + return ( + Dataset.objects.filter( + status=DatasetStatus.FINAL, + is_public=True, + is_deleted=False, + ) + .prefetch_related("keywords", "owner") + .order_by("-updated_at") + ) + + +def public_captures_queryset() -> QuerySet[Capture]: + return Capture.objects.filter(is_public=True, is_deleted=False).order_by( + "-updated_at", + ) + + +def is_federation_exportable_dataset(dataset: Dataset) -> bool: + return ( + not dataset.is_deleted + and dataset.is_public + and dataset.status == DatasetStatus.FINAL + ) + + +def is_federation_exportable_capture(capture: Capture) -> bool: + return not capture.is_deleted and capture.is_public + + +def compile_federated_dataset_doc(dataset: Dataset) -> dict[str, Any]: + """Serialize a public dataset for federation sync / OpenSearch.""" + site_name = federation_site_name() + federation_data = DatasetFederationSerializer( + dataset, + context={"site_name": site_name}, + ) + return federation_data.data + + +def compile_federated_capture_doc(capture: Capture) -> dict[str, Any]: + """Serialize a public capture for federation sync / OpenSearch.""" + site_name = federation_site_name() + federation_data = CaptureFederationSerializer( + capture, + context={"site_name": site_name}, + ) + return federation_data.data diff --git a/gateway/sds_gateway/api_methods/models.py b/gateway/sds_gateway/api_methods/models.py index 8c56b3c87..322fef020 100644 --- a/gateway/sds_gateway/api_methods/models.py +++ b/gateway/sds_gateway/api_methods/models.py @@ -88,6 +88,7 @@ class KeySources(StrEnum): SDSWebUI = "sds_web_ui" SVIBackend = "svi_backend" SVIWebUI = "svi_web_ui" + FederationSync = "federation_sync" class ItemType(StrEnum): diff --git a/gateway/sds_gateway/api_methods/permissions.py b/gateway/sds_gateway/api_methods/permissions.py new file mode 100644 index 000000000..2af3b8929 --- /dev/null +++ b/gateway/sds_gateway/api_methods/permissions.py @@ -0,0 +1,31 @@ +"""DRF permissions for API key scoping.""" + +from rest_framework.permissions import BasePermission + +from sds_gateway.api_methods.models import KeySources +from sds_gateway.users.models import UserAPIKey + + +class IsFederationSyncKey(BasePermission): + """Allow only federation sync service API keys.""" + + message = "Federation sync API key required." + + def has_permission(self, request, view) -> bool: + key = request.auth + return isinstance(key, UserAPIKey) and key.source == KeySources.FederationSync + + +class DisallowFederationSyncKey(BasePermission): + """Block federation sync keys from non-federation API routes. + + Applied globally via REST_FRAMEWORK DEFAULT_PERMISSION_CLASSES. + """ + + message = "This API key is restricted to federation export endpoints." + + def has_permission(self, request, view) -> bool: + key = request.auth + if isinstance(key, UserAPIKey) and key.source == KeySources.FederationSync: + return False + return True diff --git a/gateway/sds_gateway/api_methods/serializers/capture_serializers.py b/gateway/sds_gateway/api_methods/serializers/capture_serializers.py index fd5c4cb4d..56156b777 100644 --- a/gateway/sds_gateway/api_methods/serializers/capture_serializers.py +++ b/gateway/sds_gateway/api_methods/serializers/capture_serializers.py @@ -12,6 +12,7 @@ from rest_framework.utils.serializer_helpers import ReturnList from sds_gateway.api_methods.helpers.index_handling import retrieve_indexed_metadata +from sds_gateway.api_methods.utils.relationship_utils import get_capture_datasets from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import CaptureType from sds_gateway.api_methods.models import DEPRECATEDPostProcessedData @@ -883,3 +884,55 @@ def serialize_capture_or_composite( # Serialize as single capture serializer = CaptureGetSerializer(capture_data["capture"], context=context) return serializer.data + + +class CaptureFederationSerializer(serializers.ModelSerializer[Capture]): + """Public-safe capture payload for federation export (sync / OpenSearch).""" + + site_name = serializers.SerializerMethodField() + file_count = serializers.SerializerMethodField() + size = serializers.SerializerMethodField() + capture_props = serializers.SerializerMethodField() + dataset_ids = serializers.SerializerMethodField() + created_at = serializers.DateTimeField( + format="%Y-%m-%d %H:%M:%S%z", + read_only=True, + ) + updated_at = serializers.DateTimeField( + format="%Y-%m-%d %H:%M:%S%z", + read_only=True, + ) + + class Meta: + model = Capture + fields = [ + "uuid", + "name", + "capture_type", + "channel", + "scan_group", + "top_level_dir", + "created_at", + "updated_at", + "site_name", + "file_count", + "size", + "capture_props", + "dataset_ids", + ] + + def get_site_name(self, obj: Capture) -> str: + return str((self.context or {})["site_name"]) + + def get_file_count(self, obj: Capture) -> int: + return int(obj.get_files_summary()["total_count"]) + + def get_size(self, obj: Capture) -> int: + return int(obj.get_files_summary()["total_size"]) + + def get_capture_props(self, obj: Capture) -> dict[str, Any]: + return obj.get_opensearch_metadata() or {} + + def get_dataset_ids(self, obj: Capture) -> list[str]: + qs = get_capture_datasets(obj, include_deleted=False) + return [str(dataset.uuid) for dataset in qs] \ No newline at end of file diff --git a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py index 9f57ceffa..519a4bf79 100644 --- a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py +++ b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py @@ -334,6 +334,46 @@ class Meta: ] +class DatasetFederationSerializer(DatasetPublicSerializer): + """Serializer for dataset data for federation export.""" + + site_name = serializers.SerializerMethodField() + updated_at = serializers.DateTimeField( + format=READABLE_ISO_DATE_TIME, + read_only=True, + ) + size = serializers.SerializerMethodField() + capture_count = serializers.SerializerMethodField() + capture_file_count = serializers.SerializerMethodField() + artifact_file_count = serializers.SerializerMethodField() + + class Meta(DatasetPublicSerializer.Meta): + fields = [ + *DatasetPublicSerializer.Meta.fields, + "updated_at", + "site_name", + "size", + "capture_count", + "capture_file_count", + "artifact_file_count", + ] + + def get_site_name(self, obj: Dataset) -> str: + return str((self.context or {})["site_name"]) + + def get_size(self, obj: Dataset) -> int: + return int(obj.get_dataset_file_statistics()["total_size"]) + + def get_capture_count(self, obj: Dataset) -> int: + return obj.captures.filter(is_deleted=False).count() + + def get_capture_file_count(self, obj: Dataset) -> int: + return int(obj.get_dataset_file_statistics()["captures"]) + + def get_artifact_file_count(self, obj: Dataset) -> int: + return int(obj.get_dataset_file_statistics()["artifacts"]) + + def get_dataset_serializer( dataset: Dataset, *, diff --git a/gateway/sds_gateway/api_methods/tests/test_authenticate.py b/gateway/sds_gateway/api_methods/tests/test_authenticate.py index f070fc251..7c9620213 100644 --- a/gateway/sds_gateway/api_methods/tests/test_authenticate.py +++ b/gateway/sds_gateway/api_methods/tests/test_authenticate.py @@ -34,7 +34,7 @@ def test_auth_valid_key(self) -> None: user, auth = self.auth.authenticate(request) # Verify that the user is authenticated - assert auth is True + assert isinstance(auth, UserAPIKey) assert user is not None assert user.email == "test@example.com" diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export.py b/gateway/sds_gateway/api_methods/tests/test_federation_export.py new file mode 100644 index 000000000..1c89736a8 --- /dev/null +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export.py @@ -0,0 +1,98 @@ +"""Tests for federation export endpoints and API key scoping.""" + +from django.contrib.auth import get_user_model +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.models import KeySources +from sds_gateway.api_methods.tests.factories import CaptureFactory +from sds_gateway.api_methods.tests.factories import DatasetFactory +from sds_gateway.users.models import UserAPIKey + +User = get_user_model() + + +class FederationExportAPITest(APITestCase): + def setUp(self) -> None: + self.owner = User.objects.create(email="owner@example.com", is_approved=True) + self.sync_user = User.objects.create( + email="sync@internal.local", + is_approved=True, + ) + _obj, self.sync_key = UserAPIKey.objects.create_key( + name="sync", + user=self.sync_user, + source=KeySources.FederationSync, + ) + _obj, self.user_key = UserAPIKey.objects.create_key( + name="regular", + user=self.owner, + source=KeySources.SDSWebUI, + ) + self.public_dataset = DatasetFactory( + owner=self.owner, + is_public=True, + status=DatasetStatus.FINAL, + keywords=None, + ) + self.private_dataset = DatasetFactory( + owner=self.owner, + is_public=False, + status=DatasetStatus.DRAFT, + keywords=None, + ) + self.public_capture = CaptureFactory(owner=self.owner, is_public=True) + self.list_datasets_url = reverse( + "api:federation-export-datasets-list", + ) + self.detail_dataset_url = reverse( + "api:federation-export-dataset-detail", + kwargs={"pk": str(self.public_dataset.uuid)}, + ) + self.list_captures_url = reverse( + "api:federation-export-captures-list", + ) + + def _auth(self, key: str) -> dict[str, str]: + return {"HTTP_AUTHORIZATION": f"Api-Key: {key}"} + + def test_sync_key_can_list_public_datasets(self) -> None: + response = self.client.get(self.list_datasets_url, **self._auth(self.sync_key)) + assert response.status_code == status.HTTP_200_OK + uuids = {row["uuid"] for row in response.json()} + assert str(self.public_dataset.uuid) in uuids + assert str(self.private_dataset.uuid) not in uuids + + def test_sync_key_can_retrieve_public_dataset(self) -> None: + response = self.client.get( + self.detail_dataset_url, + **self._auth(self.sync_key), + ) + assert response.status_code == status.HTTP_200_OK + assert response.json()["uuid"] == str(self.public_dataset.uuid) + assert response.json()["site_name"] + + def test_regular_key_denied_on_export(self) -> None: + response = self.client.get(self.list_datasets_url, **self._auth(self.user_key)) + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_sync_key_denied_on_dataset_assets_api(self) -> None: + url = reverse( + "api:datasets-detail", + kwargs={"pk": str(self.public_dataset.uuid)}, + ) + response = self.client.get(url, **self._auth(self.sync_key)) + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_sync_key_denied_on_capture_list(self) -> None: + url = reverse("api:captures-list") + response = self.client.get(url, **self._auth(self.sync_key)) + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_export_captures_list(self) -> None: + response = self.client.get(self.list_captures_url, **self._auth(self.sync_key)) + assert response.status_code == status.HTTP_200_OK + uuids = {row["uuid"] for row in response.json()} + assert str(self.public_capture.uuid) in uuids diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py new file mode 100644 index 000000000..0c86277d1 --- /dev/null +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py @@ -0,0 +1,94 @@ +"""Contract tests: gateway federation export JSON ↔ sync Pydantic models.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +from django.contrib.auth import get_user_model + +from sds_gateway.api_methods.federation.export_contract import assert_field_names_match +from sds_gateway.api_methods.helpers.compile_federated_data import ( + compile_federated_capture_doc, + compile_federated_dataset_doc, +) +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.serializers.capture_serializers import ( + CaptureFederationSerializer, +) +from sds_gateway.api_methods.serializers.dataset_serializers import ( + DatasetFederationSerializer, +) +from sds_gateway.api_methods.tests.factories import CaptureFactory +from sds_gateway.api_methods.tests.factories import DatasetFactory + +_repo_root = Path(__file__).resolve().parents[4] +_federation_root = _repo_root / "federation" +if _federation_root.is_dir(): + sys.path.insert(0, str(_federation_root)) + +pytest.importorskip("sds_federation") + +from sds_federation.schemas.webhooks import ( # noqa: E402 + FederatedCaptureDoc, + FederatedDatasetDoc, +) + +User = get_user_model() + + +@pytest.mark.django_db +def test_dataset_export_field_names_match_pydantic() -> None: + owner = User.objects.create(email="owner@example.com", is_approved=True) + dataset = DatasetFactory( + owner=owner, + is_public=True, + status=DatasetStatus.FINAL, + keywords=None, + ) + serializer = DatasetFederationSerializer( + dataset, + context={"site_name": "crc"}, + ) + assert_field_names_match( + serializer, + FederatedDatasetDoc, + label="DatasetFederationSerializer", + ) + + +@pytest.mark.django_db +def test_capture_export_field_names_match_pydantic() -> None: + owner = User.objects.create(email="cap-owner@example.com", is_approved=True) + capture = CaptureFactory(owner=owner, is_public=True) + serializer = CaptureFederationSerializer( + capture, + context={"site_name": "crc"}, + ) + assert_field_names_match( + serializer, + FederatedCaptureDoc, + label="CaptureFederationSerializer", + ) + + +@pytest.mark.django_db +def test_compile_federated_dataset_doc_validates_against_pydantic() -> None: + owner = User.objects.create(email="d@example.com", is_approved=True) + dataset = DatasetFactory( + owner=owner, + is_public=True, + status=DatasetStatus.FINAL, + keywords=None, + ) + payload = compile_federated_dataset_doc(dataset) + FederatedDatasetDoc.model_validate(payload) + + +@pytest.mark.django_db +def test_compile_federated_capture_doc_validates_against_pydantic() -> None: + owner = User.objects.create(email="c@example.com", is_approved=True) + capture = CaptureFactory(owner=owner, is_public=True) + payload = compile_federated_capture_doc(capture) + FederatedCaptureDoc.model_validate(payload) diff --git a/gateway/sds_gateway/api_methods/views/capture_endpoints.py b/gateway/sds_gateway/api_methods/views/capture_endpoints.py index 94f23808e..9e2ef28a6 100644 --- a/gateway/sds_gateway/api_methods/views/capture_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/capture_endpoints.py @@ -27,7 +27,6 @@ from rest_framework.authentication import SessionAuthentication from rest_framework.decorators import action from rest_framework.pagination import PageNumberPagination -from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response @@ -208,7 +207,6 @@ class CapturePagination(PageNumberPagination): class CaptureViewSet(viewsets.ViewSet): authentication_classes = [SessionAuthentication, APIKeyAuthentication] - permission_classes = [IsAuthenticated] def _validate_and_index_metadata( self, diff --git a/gateway/sds_gateway/api_methods/views/dataset_endpoints.py b/gateway/sds_gateway/api_methods/views/dataset_endpoints.py index 0785ece85..6125fa0f0 100644 --- a/gateway/sds_gateway/api_methods/views/dataset_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/dataset_endpoints.py @@ -11,7 +11,6 @@ from rest_framework import status from rest_framework.authentication import SessionAuthentication from rest_framework.decorators import action -from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.viewsets import ViewSet @@ -56,8 +55,7 @@ def _truthy_query_param(raw: str | None) -> bool: class DatasetViewSet(ViewSet): - authentication_classes = [SessionAuthentication, APIKeyAuthentication] - permission_classes = [IsAuthenticated] + authentication_classes = [APIKeyAuthentication] def _get_file_objects( self, diff --git a/gateway/sds_gateway/api_methods/views/federation_endpoints.py b/gateway/sds_gateway/api_methods/views/federation_endpoints.py new file mode 100644 index 000000000..2f9c06016 --- /dev/null +++ b/gateway/sds_gateway/api_methods/views/federation_endpoints.py @@ -0,0 +1,87 @@ +"""Federation export API (sync service → gateway, Postgres public metadata).""" + +from __future__ import annotations + +from django.shortcuts import get_object_or_404 +from drf_spectacular.utils import extend_schema +from rest_framework.decorators import action +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.viewsets import ViewSet + +from sds_gateway.api_methods.authentication import APIKeyAuthentication +from sds_gateway.api_methods.helpers.compile_federated_data import ( + compile_federated_capture_doc, +) +from sds_gateway.api_methods.helpers.compile_federated_data import ( + compile_federated_dataset_doc, +) +from sds_gateway.api_methods.helpers.compile_federated_data import ( + is_federation_exportable_capture, +) +from sds_gateway.api_methods.helpers.compile_federated_data import ( + is_federation_exportable_dataset, +) +from sds_gateway.api_methods.helpers.compile_federated_data import ( + public_captures_queryset, +) +from sds_gateway.api_methods.helpers.compile_federated_data import ( + public_datasets_queryset, +) +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.permissions import IsFederationSyncKey + + +@extend_schema(exclude=True) +class FederationViewSet(ViewSet): + """Internal export endpoints for the federation sync service.""" + + authentication_classes = [APIKeyAuthentication] + permission_classes = [IsFederationSyncKey] + + @action(detail=False, methods=["get"], url_path="export/datasets") + def export_datasets_list(self, request: Request) -> Response: + """List all public finalized datasets for federation bootstrap.""" + datasets = public_datasets_queryset() + return Response( + [compile_federated_dataset_doc(dataset) for dataset in datasets], + ) + + @action( + detail=False, + methods=["get"], + url_path=r"export/datasets/(?P[^/.]+)", + ) + def export_dataset_detail(self, request: Request, pk: str | None = None) -> Response: + """Return one public dataset for sync after a local Redis event.""" + dataset = get_object_or_404(Dataset, pk=pk, is_deleted=False) + if not is_federation_exportable_dataset(dataset): + return Response( + {"detail": "Dataset is not available for federation export."}, + status=404, + ) + return Response(compile_federated_dataset_doc(dataset)) + + @action(detail=False, methods=["get"], url_path="export/captures") + def export_captures_list(self, request: Request) -> Response: + """List all public captures for federation bootstrap.""" + captures = public_captures_queryset() + return Response( + [compile_federated_capture_doc(capture) for capture in captures], + ) + + @action( + detail=False, + methods=["get"], + url_path=r"export/captures/(?P[^/.]+)", + ) + def export_capture_detail(self, request: Request, pk: str | None = None) -> Response: + """Return one public capture for sync after a local Redis event.""" + capture = get_object_or_404(Capture, pk=pk, is_deleted=False) + if not is_federation_exportable_capture(capture): + return Response( + {"detail": "Capture is not available for federation export."}, + status=404, + ) + return Response(compile_federated_capture_doc(capture)) diff --git a/gateway/sds_gateway/api_methods/views/file_endpoints.py b/gateway/sds_gateway/api_methods/views/file_endpoints.py index 0a441566b..1e9308df8 100644 --- a/gateway/sds_gateway/api_methods/views/file_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/file_endpoints.py @@ -24,7 +24,6 @@ from rest_framework import status from rest_framework.decorators import action from rest_framework.pagination import PageNumberPagination -from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response from rest_framework.views import APIView @@ -65,7 +64,6 @@ class FilePagination(PageNumberPagination): class FileViewSet(ViewSet): authentication_classes = [APIKeyAuthentication] - permission_classes = [IsAuthenticated] @staticmethod def _paginated_list_response( @@ -698,7 +696,6 @@ def download_file(self, request: Request, pk: str | None = None) -> HttpResponse class CheckFileContentsExistView(APIView): authentication_classes = [APIKeyAuthentication] - permission_classes = [IsAuthenticated] @extend_schema( request=FilePostSerializer, diff --git a/gateway/sds_gateway/users/management/commands/create_federation_sync_api_key.py b/gateway/sds_gateway/users/management/commands/create_federation_sync_api_key.py new file mode 100644 index 000000000..2c7ede4e2 --- /dev/null +++ b/gateway/sds_gateway/users/management/commands/create_federation_sync_api_key.py @@ -0,0 +1,46 @@ +"""Create the federation sync service user and API key.""" + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand +from loguru import logger as log + +from sds_gateway.api_methods.models import KeySources +from sds_gateway.users.models import UserAPIKey + + +class Command(BaseCommand): + """Provision federation-sync@internal user and a FederationSync API key.""" + + help = "Create federation sync service user and API key (prints raw key once)." + + def handle(self, *args, **options) -> None: + user_model = get_user_model() + email = settings.FEDERATION_SYNC_USER_EMAIL + + user, created = user_model.objects.get_or_create( + email=email, + defaults={ + "is_active": True, + "is_approved": True, + }, + ) + if created: + user.set_unusable_password() + user.save(update_fields=["password"]) + log.info("Created federation sync user {}", email) + else: + log.info("Using existing federation sync user {}", email) + + UserAPIKey.objects.filter( + user=user, + source=KeySources.FederationSync, + ).delete() + + _obj, raw_key = UserAPIKey.objects.create_key( + name="federation-sync", + user=user, + source=KeySources.FederationSync, + description="Federation sync service (export endpoints only)", + ) + self.stdout.write(self.style.SUCCESS(f"Federation sync API key: {raw_key}")) diff --git a/gateway/sds_gateway/users/migrations/0012_alter_userapikey_source_federation_sync.py b/gateway/sds_gateway/users/migrations/0012_alter_userapikey_source_federation_sync.py new file mode 100644 index 000000000..084b10c0b --- /dev/null +++ b/gateway/sds_gateway/users/migrations/0012_alter_userapikey_source_federation_sync.py @@ -0,0 +1,38 @@ +# Generated manually for FederationSync API key source + +from django.db import migrations, models + +import sds_gateway.api_methods.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0011_user_orcid_id_alter_user_is_approved"), + ] + + operations = [ + migrations.AlterField( + model_name="userapikey", + name="source", + field=models.CharField( + choices=[ + (sds_gateway.api_methods.models.KeySources["SDSWebUI"], "SDS Web UI"), + ( + sds_gateway.api_methods.models.KeySources["SVIBackend"], + "SVI Backend", + ), + ( + sds_gateway.api_methods.models.KeySources["SVIWebUI"], + "SVI Web UI", + ), + ( + sds_gateway.api_methods.models.KeySources["FederationSync"], + "Federation Sync", + ), + ], + default=sds_gateway.api_methods.models.KeySources["SDSWebUI"], + max_length=255, + ), + ), + ] diff --git a/gateway/sds_gateway/users/migrations/max_migration.txt b/gateway/sds_gateway/users/migrations/max_migration.txt index fda121cc0..7707335c4 100644 --- a/gateway/sds_gateway/users/migrations/max_migration.txt +++ b/gateway/sds_gateway/users/migrations/max_migration.txt @@ -1 +1 @@ -0011_user_orcid_id_alter_user_is_approved +0012_alter_userapikey_source_federation_sync \ No newline at end of file diff --git a/gateway/sds_gateway/users/models.py b/gateway/sds_gateway/users/models.py index b1a31d0db..f5b23370f 100644 --- a/gateway/sds_gateway/users/models.py +++ b/gateway/sds_gateway/users/models.py @@ -64,6 +64,7 @@ class UserAPIKey(AbstractAPIKey): (KeySources.SDSWebUI, "SDS Web UI"), (KeySources.SVIBackend, "SVI Backend"), (KeySources.SVIWebUI, "SVI Web UI"), + (KeySources.FederationSync, "Federation Sync"), ] user = cast( "User", From 7b6f1e0d9d2cfffc73ceadce54669f9b85e9e36e Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 19 Jun 2026 11:39:01 -0400 Subject: [PATCH 02/17] federation config hardening --- gateway/.envs/example/django.env | 14 +- gateway/config/settings/base.py | 49 +++++ gateway/sds_gateway/api_methods/apps.py | 5 + .../api_methods/federation/availability.py | 175 +++++++++++++++ .../api_methods/federation/events.py | 4 + .../api_methods/federation/permissions.py | 44 ++++ .../api_methods/federation/signals.py | 26 ++- .../tests/test_federation_export.py | 26 ++- .../tests/test_federation_hardening.py | 205 ++++++++++++++++++ .../api_methods/views/federation_endpoints.py | 8 +- 10 files changed, 537 insertions(+), 19 deletions(-) create mode 100644 gateway/sds_gateway/api_methods/federation/availability.py create mode 100644 gateway/sds_gateway/api_methods/federation/permissions.py create mode 100644 gateway/sds_gateway/api_methods/tests/test_federation_hardening.py diff --git a/gateway/.envs/example/django.env b/gateway/.envs/example/django.env index 77e2990f8..356c45345 100644 --- a/gateway/.envs/example/django.env +++ b/gateway/.envs/example/django.env @@ -12,12 +12,16 @@ API_KEY= # ------------------------------------------------------------------------------ # the below environment variables are used for site federation with peers. # By default, federation events are disabled. -# If FEDERATION_EVENTS_ENABLED is TRUE, deployment will need to include +# If FEDERATION_ENABLED is TRUE, deployment will need to include # federation sync service configuration. -FEDERATION_SITE_NAME= -FEDERATION_EVENTS_ENABLED=false -FEDERATION_EVENTS_CHANNEL=federation:events -FEDERATION_SYNC_USER_EMAIL= +# FEDERATION_ENABLED=true +# FEDERATION_EVENTS_ENABLED=true +# FEDERATION_EVENTS_CHANNEL=federation:events +# FEDERATION_SYNC_USER_EMAIL= +# FEDERATION_SITE_NAME= +# FEDERATION_SYNC_HEALTH_URL=http://federation-sync:8000/sync/health +# FEDERATION_EXPORT_INTERNAL_HEADER_SECRET= +# FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR=false # AUTH0 # ------------------------------------------------------------------------------ diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index 8c483a22d..20ceef92b 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -716,6 +716,8 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: "FEDERATION_SITE_NAME", default="", ) +# Master switch: when False, federation export and Redis events are inactive. +FEDERATION_ENABLED: bool = env.bool("FEDERATION_ENABLED", default=False) FEDERATION_EVENTS_ENABLED: bool = env.bool("FEDERATION_EVENTS_ENABLED", default=False) FEDERATION_EVENTS_CHANNEL: str = env.str( "FEDERATION_EVENTS_CHANNEL", @@ -725,6 +727,53 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: "FEDERATION_SYNC_USER_EMAIL", default="federation-sync@internal.local", ) +# Sync service health (e.g. http://federation-sync:8000/sync/health). +FEDERATION_SYNC_HEALTH_URL: str = env.str("FEDERATION_SYNC_HEALTH_URL", default="") +FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT: float = env.float( + "FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT", + default=2.0, +) +FEDERATION_SKIP_SYNC_HEALTH_PROBE: bool = env.bool( + "FEDERATION_SKIP_SYNC_HEALTH_PROBE", + default=False, +) +FEDERATION_SKIP_SYNC_API_KEY_CHECK: bool = env.bool( + "FEDERATION_SKIP_SYNC_API_KEY_CHECK", + default=False, +) +FEDERATION_SKIP_REDIS_PROBE: bool = env.bool( + "FEDERATION_SKIP_REDIS_PROBE", + default=False, +) +# Set at startup / periodic recheck by federation.availability. +FEDERATION_OPERATIONAL: bool = False +FEDERATION_OPERATIONAL_REASON: str = "" +# Tests may set via override_settings without running probes. +FEDERATION_OPERATIONAL_OVERRIDE: bool | None = None +# Export API: allow only these source CIDRs (sync container / internal mesh). +FEDERATION_EXPORT_ALLOWED_CIDRS: list[str] = env.list( + "FEDERATION_EXPORT_ALLOWED_CIDRS", + default=[ + "127.0.0.1/32", + "::1/128", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + ], +) +FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR: bool = env.bool( + "FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR", + default=False, +) +# Optional: Traefik injects this on internal routes only (empty = IP check only). +FEDERATION_EXPORT_INTERNAL_HEADER_NAME: str = env.str( + "FEDERATION_EXPORT_INTERNAL_HEADER_NAME", + default="X-SDS-Federation-Internal", +) +FEDERATION_EXPORT_INTERNAL_HEADER_SECRET: str = env.str( + "FEDERATION_EXPORT_INTERNAL_HEADER_SECRET", + default="", +) # ADMIN_CONSOLE_ENV is used to visually distinguish between different environments # (production, staging, local) in the admin console and error emails. It does not affect diff --git a/gateway/sds_gateway/api_methods/apps.py b/gateway/sds_gateway/api_methods/apps.py index 2c38194ff..442f6f003 100644 --- a/gateway/sds_gateway/api_methods/apps.py +++ b/gateway/sds_gateway/api_methods/apps.py @@ -14,6 +14,11 @@ class ApiMethodsConfig(AppConfig): def ready(self) -> None: import sds_gateway.api_methods.federation.signals # noqa: F401 import sds_gateway.api_methods.schema # noqa: F401 + from sds_gateway.api_methods.federation.availability import ( + initialize_federation_operational_state, + ) + + initialize_federation_operational_state() silence_unwanted_logs() diff --git a/gateway/sds_gateway/api_methods/federation/availability.py b/gateway/sds_gateway/api_methods/federation/availability.py new file mode 100644 index 000000000..925ef7512 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/availability.py @@ -0,0 +1,175 @@ +"""Federation operational status: config, sync health, Redis, sync API key.""" + +from __future__ import annotations + +import ipaddress +import json +import secrets +import time +import urllib.error +import urllib.request +from typing import Any + +from django.conf import settings +from loguru import logger as log + +from sds_gateway.api_methods.models import KeySources +from sds_gateway.users.models import UserAPIKey + +_RECHECK_INTERVAL_SECONDS = 60.0 +_last_evaluated_at: float = 0.0 +_cached_operational: bool = False +_cached_reason: str = "not evaluated" + + +def _setting(name: str, default: Any = None) -> Any: + return getattr(settings, name, default) + + +def _parse_cidrs(raw: list[str]) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]: + networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] + for item in raw: + networks.append(ipaddress.ip_network(item.strip(), strict=False)) + return networks + + +def federation_client_ip(request) -> str | None: + """Resolve client IP for federation export access control.""" + trust_forwarded = _setting("FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR", False) + if trust_forwarded: + forwarded = request.META.get("HTTP_X_FORWARDED_FOR", "") + if forwarded: + return forwarded.split(",")[0].strip() + remote = request.META.get("REMOTE_ADDR") + if remote: + return str(remote).strip() + return None + + +def is_client_ip_allowed_for_federation_export(request) -> bool: + cidrs = _parse_cidrs(_setting("FEDERATION_EXPORT_ALLOWED_CIDRS", [])) + if not cidrs: + return False + client_ip = federation_client_ip(request) + if not client_ip: + return False + try: + addr = ipaddress.ip_address(client_ip) + except ValueError: + return False + return any(addr in network for network in cidrs) + + +def is_federation_internal_header_valid(request) -> bool: + secret = _setting("FEDERATION_EXPORT_INTERNAL_HEADER_SECRET", "") + if not secret: + return True + header_name = _setting( + "FEDERATION_EXPORT_INTERNAL_HEADER_NAME", + "X-SDS-Federation-Internal", + ) + meta_key = "HTTP_" + header_name.upper().replace("-", "_") + provided = request.META.get(meta_key, "") + if not provided: + return False + return secrets.compare_digest(str(provided), str(secret)) + + +def _sync_health_ok() -> tuple[bool, str]: + if _setting("FEDERATION_SKIP_SYNC_HEALTH_PROBE", False): + return True, "health probe skipped" + url = (_setting("FEDERATION_SYNC_HEALTH_URL") or "").strip() + if not url: + return False, "FEDERATION_SYNC_HEALTH_URL is not set" + timeout = float(_setting("FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT", 2.0)) + request = urllib.request.Request(url, method="GET") + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + if response.status != 200: + return False, f"sync health returned HTTP {response.status}" + body = response.read().decode("utf-8", errors="replace") + except urllib.error.URLError as exc: + return False, f"sync health probe failed: {exc.reason}" + except TimeoutError: + return False, "sync health probe timed out" + if body: + try: + payload = json.loads(body) + if isinstance(payload, dict) and payload.get("status") == "ok": + return True, "sync health ok" + except json.JSONDecodeError: + pass + return True, "sync health returned 200" + return True, "sync health returned 200" + + +def _sync_api_key_present() -> tuple[bool, str]: + if _setting("FEDERATION_SKIP_SYNC_API_KEY_CHECK", False): + return True, "sync API key check skipped" + exists = UserAPIKey.objects.filter(source=KeySources.FederationSync).exists() + if not exists: + return False, "no FederationSync API key in database" + return True, "FederationSync API key present" + + +def _redis_ok() -> tuple[bool, str]: + if not _setting("FEDERATION_EVENTS_ENABLED", False): + return True, "redis not required (events disabled)" + if _setting("FEDERATION_SKIP_REDIS_PROBE", False): + return True, "redis probe skipped" + from sds_gateway.api_methods.tasks import get_redis_client + + try: + client = get_redis_client() + client.ping() + except Exception as exc: # noqa: BLE001 + return False, f"redis ping failed: {exc}" + return True, "redis ok" + + +def evaluate_federation_operational() -> tuple[bool, str]: + if not _setting("FEDERATION_ENABLED", False): + return False, "FEDERATION_ENABLED is False" + + for check in (_sync_api_key_present, _sync_health_ok, _redis_ok): + ok, reason = check() + if not ok: + return False, reason + return True, "federation operational" + + +def refresh_federation_operational_state(*, force: bool = False) -> tuple[bool, str]: + global _cached_operational, _cached_reason, _last_evaluated_at + + now = time.monotonic() + if ( + not force + and _last_evaluated_at + and (now - _last_evaluated_at) < _RECHECK_INTERVAL_SECONDS + ): + return _cached_operational, _cached_reason + + operational, reason = evaluate_federation_operational() + _cached_operational = operational + _cached_reason = reason + _last_evaluated_at = now + settings.FEDERATION_OPERATIONAL = operational + settings.FEDERATION_OPERATIONAL_REASON = reason + return operational, reason + + +def initialize_federation_operational_state() -> None: + operational, reason = refresh_federation_operational_state(force=True) + if operational: + log.info("Federation is operational: {}", reason) + else: + log.warning("Federation disabled: {}", reason) + + +def is_federation_operational() -> bool: + if _setting("FEDERATION_OPERATIONAL_OVERRIDE", None) is not None: + return bool(_setting("FEDERATION_OPERATIONAL_OVERRIDE")) + if not _setting("FEDERATION_ENABLED", False): + return False + operational, _reason = refresh_federation_operational_state() + return operational diff --git a/gateway/sds_gateway/api_methods/federation/events.py b/gateway/sds_gateway/api_methods/federation/events.py index f1d8ff897..7e90af3a8 100644 --- a/gateway/sds_gateway/api_methods/federation/events.py +++ b/gateway/sds_gateway/api_methods/federation/events.py @@ -11,6 +11,7 @@ from django.conf import settings from loguru import logger as log +from sds_gateway.api_methods.federation.availability import is_federation_operational from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.tasks import get_redis_client @@ -25,6 +26,9 @@ def publish_federation_event( timestamp: datetime | None = None, ) -> None: """Notify the local federation sync service via Redis pub/sub.""" + if not is_federation_operational(): + log.debug("Federation not operational, skipping Redis publish") + return channel = getattr(settings, "FEDERATION_EVENTS_CHANNEL", "federation:events") payload: dict[str, Any] = { "event_type": event_type, diff --git a/gateway/sds_gateway/api_methods/federation/permissions.py b/gateway/sds_gateway/api_methods/federation/permissions.py new file mode 100644 index 000000000..61f2ecad8 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/permissions.py @@ -0,0 +1,44 @@ +"""Permissions specific to federation export endpoints.""" + +from django.conf import settings +from rest_framework.exceptions import APIException +from rest_framework.permissions import BasePermission + +from sds_gateway.api_methods.federation.availability import ( + is_client_ip_allowed_for_federation_export, +) +from sds_gateway.api_methods.federation.availability import ( + is_federation_internal_header_valid, +) +from sds_gateway.api_methods.federation.availability import is_federation_operational + + +class FederationNotOperational(APIException): + status_code = 503 + default_detail = "Federation is not configured or the sync service is unavailable." + default_code = "federation_unavailable" + + +class IsFederationOperational(BasePermission): + """Deny export when federation failed startup / health checks.""" + + message = FederationNotOperational.default_detail + + def has_permission(self, request, view) -> bool: + if not is_federation_operational(): + detail = getattr(settings, "FEDERATION_OPERATIONAL_REASON", "") or ( + FederationNotOperational.default_detail + ) + raise FederationNotOperational(detail=detail) + return True + + +class IsFederationInternalExportClient(BasePermission): + """Restrict export to internal networks (and optional edge-injected header).""" + + message = "Federation export is only available to internal clients." + + def has_permission(self, request, view) -> bool: + if not is_client_ip_allowed_for_federation_export(request): + return False + return is_federation_internal_header_valid(request) diff --git a/gateway/sds_gateway/api_methods/federation/signals.py b/gateway/sds_gateway/api_methods/federation/signals.py index c65619971..d235aae58 100644 --- a/gateway/sds_gateway/api_methods/federation/signals.py +++ b/gateway/sds_gateway/api_methods/federation/signals.py @@ -7,6 +7,7 @@ from django.dispatch import receiver from loguru import logger as log +from sds_gateway.api_methods.federation.availability import is_federation_operational from sds_gateway.api_methods.federation.events import publish_federation_event from sds_gateway.api_methods.helpers.compile_federated_data import ( is_federation_exportable_capture, @@ -24,6 +25,19 @@ def _event_type(*, created: bool, exportable: bool) -> str: return "deleted" return "created" if created else "updated" +def _skip_signal() -> bool: + if not getattr(settings, "FEDERATION_EVENTS_ENABLED", False): + log.debug( + "FEDERATION_EVENTS_ENABLED is False, " + "skipping federation signal", + ) + return True + + if not is_federation_operational(): + log.debug("Federation not operational, skipping signal") + return True + + return False @receiver(post_save, sender=Dataset) def federation_dataset_changed( @@ -32,11 +46,7 @@ def federation_dataset_changed( created: bool, **kwargs, ) -> None: - if not getattr(settings, "FEDERATION_EVENTS_ENABLED", True): - log.debug( - "FEDERATION_EVENTS_ENABLED is False, " - "skipping federation dataset changed signal", - ) + if _skip_signal(): return exportable = is_federation_exportable_dataset(instance) @@ -55,11 +65,7 @@ def federation_capture_changed( created: bool, **kwargs, ) -> None: - if not getattr(settings, "FEDERATION_EVENTS_ENABLED", True): - log.debug( - "FEDERATION_EVENTS_ENABLED is False, " - "skipping federation capture changed signal", - ) + if _skip_signal(): return exportable = is_federation_exportable_capture(instance) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export.py b/gateway/sds_gateway/api_methods/tests/test_federation_export.py index 1c89736a8..7d68dae5f 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export.py @@ -1,6 +1,7 @@ """Tests for federation export endpoints and API key scoping.""" from django.contrib.auth import get_user_model +from django.test import override_settings from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase @@ -14,6 +15,12 @@ User = get_user_model() +@override_settings( + FEDERATION_ENABLED=True, + FEDERATION_OPERATIONAL_OVERRIDE=True, + FEDERATION_EXPORT_ALLOWED_CIDRS=["0.0.0.0/0", "::/0"], + FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="", +) class FederationExportAPITest(APITestCase): def setUp(self) -> None: self.owner = User.objects.create(email="owner@example.com", is_approved=True) @@ -59,7 +66,11 @@ def _auth(self, key: str) -> dict[str, str]: return {"HTTP_AUTHORIZATION": f"Api-Key: {key}"} def test_sync_key_can_list_public_datasets(self) -> None: - response = self.client.get(self.list_datasets_url, **self._auth(self.sync_key)) + response = self.client.get( + self.list_datasets_url, + REMOTE_ADDR="127.0.0.1", + **self._auth(self.sync_key), + ) assert response.status_code == status.HTTP_200_OK uuids = {row["uuid"] for row in response.json()} assert str(self.public_dataset.uuid) in uuids @@ -68,6 +79,7 @@ def test_sync_key_can_list_public_datasets(self) -> None: def test_sync_key_can_retrieve_public_dataset(self) -> None: response = self.client.get( self.detail_dataset_url, + REMOTE_ADDR="127.0.0.1", **self._auth(self.sync_key), ) assert response.status_code == status.HTTP_200_OK @@ -75,7 +87,11 @@ def test_sync_key_can_retrieve_public_dataset(self) -> None: assert response.json()["site_name"] def test_regular_key_denied_on_export(self) -> None: - response = self.client.get(self.list_datasets_url, **self._auth(self.user_key)) + response = self.client.get( + self.list_datasets_url, + REMOTE_ADDR="127.0.0.1", + **self._auth(self.user_key), + ) assert response.status_code == status.HTTP_403_FORBIDDEN def test_sync_key_denied_on_dataset_assets_api(self) -> None: @@ -92,7 +108,11 @@ def test_sync_key_denied_on_capture_list(self) -> None: assert response.status_code == status.HTTP_403_FORBIDDEN def test_export_captures_list(self) -> None: - response = self.client.get(self.list_captures_url, **self._auth(self.sync_key)) + response = self.client.get( + self.list_captures_url, + REMOTE_ADDR="127.0.0.1", + **self._auth(self.sync_key), + ) assert response.status_code == status.HTTP_200_OK uuids = {row["uuid"] for row in response.json()} assert str(self.public_capture.uuid) in uuids diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py new file mode 100644 index 000000000..bc2c97b14 --- /dev/null +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -0,0 +1,205 @@ +"""Tests for federation operational checks and export access control.""" + +from __future__ import annotations + +from unittest.mock import MagicMock +from unittest.mock import patch + +import pytest +from django.test import RequestFactory +from django.test import override_settings +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from sds_gateway.api_methods.federation.availability import evaluate_federation_operational +from sds_gateway.api_methods.federation.availability import ( + is_client_ip_allowed_for_federation_export, +) +from sds_gateway.api_methods.federation.availability import ( + is_federation_internal_header_valid, +) +from sds_gateway.api_methods.federation.availability import refresh_federation_operational_state +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.models import KeySources +from sds_gateway.api_methods.tests.factories import DatasetFactory +from sds_gateway.users.models import UserAPIKey + +pytestmark = pytest.mark.django_db + + +class TestFederationAvailability: + @override_settings(FEDERATION_ENABLED=False) + def test_disabled_when_master_switch_off(self) -> None: + ok, reason = evaluate_federation_operational() + assert ok is False + assert "FEDERATION_ENABLED" in reason + + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, + FEDERATION_SKIP_REDIS_PROBE=True, + ) + def test_operational_when_probes_skipped(self) -> None: + ok, reason = evaluate_federation_operational() + assert ok is True + + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, + FEDERATION_SKIP_REDIS_PROBE=True, + ) + def test_fails_without_sync_api_key(self) -> None: + ok, reason = evaluate_federation_operational() + assert ok is False + assert "FederationSync" in reason + + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SYNC_HEALTH_URL="http://sync.test/health", + FEDERATION_SKIP_REDIS_PROBE=True, + ) + @patch("sds_gateway.api_methods.federation.availability.urllib.request.urlopen") + def test_health_probe_success(self, mock_urlopen: MagicMock) -> None: + response = MagicMock() + response.status = 200 + response.read.return_value = b'{"status":"ok"}' + response.__enter__.return_value = response + response.__exit__.return_value = None + mock_urlopen.return_value = response + + ok, _reason = evaluate_federation_operational() + assert ok is True + + @override_settings( + FEDERATION_EXPORT_ALLOWED_CIDRS=["10.0.0.0/8"], + ) + def test_client_ip_allowlist(self) -> None: + factory = RequestFactory() + allowed = factory.get("/", REMOTE_ADDR="10.1.2.3") + denied = factory.get("/", REMOTE_ADDR="203.0.113.8") + assert is_client_ip_allowed_for_federation_export(allowed) is True + assert is_client_ip_allowed_for_federation_export(denied) is False + + @override_settings( + FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="top-secret", + FEDERATION_EXPORT_INTERNAL_HEADER_NAME="X-SDS-Federation-Internal", + ) + def test_internal_header_when_configured(self) -> None: + factory = RequestFactory() + missing = factory.get("/") + valid = factory.get("/", HTTP_X_SDS_FEDERATION_INTERNAL="top-secret") + invalid = factory.get("/", HTTP_X_SDS_FEDERATION_INTERNAL="wrong") + assert is_federation_internal_header_valid(missing) is False + assert is_federation_internal_header_valid(valid) is True + assert is_federation_internal_header_valid(invalid) is False + + @override_settings(FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="") + def test_internal_header_optional_when_secret_empty(self) -> None: + request = RequestFactory().get("/") + assert is_federation_internal_header_valid(request) is True + + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, + FEDERATION_SKIP_REDIS_PROBE=True, + ) + def test_refresh_sets_settings_flags(self) -> None: + operational, reason = refresh_federation_operational_state(force=True) + from django.conf import settings + + assert operational is True + assert settings.FEDERATION_OPERATIONAL is True + assert reason + + +@override_settings( + FEDERATION_ENABLED=True, + FEDERATION_OPERATIONAL_OVERRIDE=True, + FEDERATION_EXPORT_ALLOWED_CIDRS=["0.0.0.0/0", "::/0"], + FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="", +) +class FederationExportAccessControlTest(APITestCase): + """Export API with operational + network permissions enabled for tests.""" + + def setUp(self) -> None: + from django.contrib.auth import get_user_model + + User = get_user_model() + self.sync_user = User.objects.create( + email="sync@internal.local", + is_approved=True, + ) + _obj, self.sync_key = UserAPIKey.objects.create_key( + name="sync", + user=self.sync_user, + source=KeySources.FederationSync, + ) + owner = User.objects.create(email="owner@example.com", is_approved=True) + self.public_dataset = DatasetFactory( + owner=owner, + is_public=True, + status=DatasetStatus.FINAL, + keywords=None, + ) + self.list_datasets_url = reverse("api:federation-export-datasets-list") + + def _auth(self, key: str) -> dict[str, str]: + return {"HTTP_AUTHORIZATION": f"Api-Key: {key}"} + + def test_sync_key_allowed_from_loopback(self) -> None: + response = self.client.get( + self.list_datasets_url, + REMOTE_ADDR="127.0.0.1", + **self._auth(self.sync_key), + ) + assert response.status_code == status.HTTP_200_OK + + @override_settings( + FEDERATION_EXPORT_ALLOWED_CIDRS=["10.0.0.0/8"], + FEDERATION_OPERATIONAL_OVERRIDE=True, + ) + def test_sync_key_denied_from_public_ip(self) -> None: + response = self.client.get( + self.list_datasets_url, + REMOTE_ADDR="203.0.113.1", + **self._auth(self.sync_key), + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + + @override_settings( + FEDERATION_OPERATIONAL_OVERRIDE=False, + FEDERATION_OPERATIONAL_REASON="sync health probe failed", + FEDERATION_EXPORT_ALLOWED_CIDRS=["0.0.0.0/0"], + ) + def test_export_returns_503_when_not_operational(self) -> None: + response = self.client.get( + self.list_datasets_url, + REMOTE_ADDR="127.0.0.1", + **self._auth(self.sync_key), + ) + assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + @override_settings( + FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="edge-secret", + FEDERATION_OPERATIONAL_OVERRIDE=True, + FEDERATION_EXPORT_ALLOWED_CIDRS=["127.0.0.1/32"], + ) + def test_internal_header_required_when_configured(self) -> None: + denied = self.client.get( + self.list_datasets_url, + REMOTE_ADDR="127.0.0.1", + **self._auth(self.sync_key), + ) + assert denied.status_code == status.HTTP_403_FORBIDDEN + + allowed = self.client.get( + self.list_datasets_url, + REMOTE_ADDR="127.0.0.1", + HTTP_X_SDS_FEDERATION_INTERNAL="edge-secret", + **self._auth(self.sync_key), + ) + assert allowed.status_code == status.HTTP_200_OK diff --git a/gateway/sds_gateway/api_methods/views/federation_endpoints.py b/gateway/sds_gateway/api_methods/views/federation_endpoints.py index 2f9c06016..087f3472d 100644 --- a/gateway/sds_gateway/api_methods/views/federation_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/federation_endpoints.py @@ -30,6 +30,8 @@ ) from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.federation.permissions import IsFederationInternalExportClient +from sds_gateway.api_methods.federation.permissions import IsFederationOperational from sds_gateway.api_methods.permissions import IsFederationSyncKey @@ -38,7 +40,11 @@ class FederationViewSet(ViewSet): """Internal export endpoints for the federation sync service.""" authentication_classes = [APIKeyAuthentication] - permission_classes = [IsFederationSyncKey] + permission_classes = [ + IsFederationSyncKey, + IsFederationOperational, + IsFederationInternalExportClient, + ] @action(detail=False, methods=["get"], url_path="export/datasets") def export_datasets_list(self, request: Request) -> Response: From 9e3fc32191c912fe0f4567482b7b42bd8b588691 Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 19 Jun 2026 13:48:04 -0400 Subject: [PATCH 03/17] pre-commit fixes --- gateway/config/api_router.py | 2 +- gateway/pyproject.toml | 4 ++ gateway/scripts/fallow-cross-file-dupes.sh | 33 +++++++----- gateway/sds_gateway/api_methods/apps.py | 2 +- .../api_methods/federation/availability.py | 51 ++++++++++--------- .../api_methods/federation/events.py | 8 ++- .../api_methods/federation/export_contract.py | 9 +++- .../api_methods/federation/signals.py | 11 ++-- .../helpers/compile_federated_data.py | 11 ++-- .../sds_gateway/api_methods/permissions.py | 6 +-- .../serializers/capture_serializers.py | 3 +- .../tests/test_federation_export_contract.py | 8 +-- .../tests/test_federation_hardening.py | 22 ++++---- .../api_methods/views/federation_endpoints.py | 20 ++++++-- 14 files changed, 114 insertions(+), 76 deletions(-) diff --git a/gateway/config/api_router.py b/gateway/config/api_router.py index 976ff5023..b181398ea 100644 --- a/gateway/config/api_router.py +++ b/gateway/config/api_router.py @@ -5,9 +5,9 @@ from sds_gateway.api_methods.views.auth_endpoints import ValidateAuthViewSet from sds_gateway.api_methods.views.capture_endpoints import CaptureViewSet from sds_gateway.api_methods.views.dataset_endpoints import DatasetViewSet +from sds_gateway.api_methods.views.federation_endpoints import FederationViewSet from sds_gateway.api_methods.views.file_endpoints import FileViewSet from sds_gateway.api_methods.views.file_endpoints import check_contents_exist -from sds_gateway.api_methods.views.federation_endpoints import FederationViewSet from sds_gateway.users.api.views import UserViewSet from sds_gateway.visualizations.api_views import VisualizationViewSet diff --git a/gateway/pyproject.toml b/gateway/pyproject.toml index 4ed42b8b8..a88f1bdee 100644 --- a/gateway/pyproject.toml +++ b/gateway/pyproject.toml @@ -147,6 +147,10 @@ # https://deptry.com/usage/#per-rule-ignores [tool.deptry.per_rule_ignores] + DEP001 = [ + # optional monorepo sibling; contract tests add federation/ to sys.path + "sds_federation", + ] DEP002 = [ # packages that are installed but not imported "argon2-cffi", # used by django for argon2 password hashing diff --git a/gateway/scripts/fallow-cross-file-dupes.sh b/gateway/scripts/fallow-cross-file-dupes.sh index 5d2e91c14..c1ac45594 100755 --- a/gateway/scripts/fallow-cross-file-dupes.sh +++ b/gateway/scripts/fallow-cross-file-dupes.sh @@ -2,24 +2,31 @@ set -euo pipefail cd "$(dirname "$0")/.." -# Use first available: vpx > bunx > npx -if command -v pnpx &>/dev/null; then - RUNNER=pnpx -elif command -v vpx &>/dev/null; then - RUNNER=vpx -elif command -v bunx &>/dev/null; then - RUNNER=bunx -elif command -v npx &>/dev/null; then - RUNNER=npx +LOCAL_FALLOW="node_modules/.bin/fallow" +if [[ -x "${LOCAL_FALLOW}" ]]; then + "${LOCAL_FALLOW}" dupes --format json -q | jq -e ' + [ (.clone_groups // .dupes.clone_groups // [])[] + | select((.instances | map(.file) | unique | length) > 1) + ] | length == 0 +' >/dev/null else - echo "Error: neither vpx, bunx, nor npx found in PATH" >&2 - exit 1 -fi + # Use first available: vpx > bunx > npx (avoid pnpx; global pnpm may need newer Node) + if command -v vpx &>/dev/null; then + RUNNER=(vpx) + elif command -v bunx &>/dev/null; then + RUNNER=(bunx) + elif command -v npx &>/dev/null; then + RUNNER=(npx) + else + echo "Error: neither local fallow, vpx, bunx, nor npx found" >&2 + exit 1 + fi -"${RUNNER}" fallow dupes --format json -q | jq -e ' + "${RUNNER[@]}" fallow dupes --format json -q | jq -e ' [ (.clone_groups // .dupes.clone_groups // [])[] | select((.instances | map(.file) | unique | length) > 1) ] | length == 0 ' >/dev/null +fi echo "No cross-file clone groups detected." diff --git a/gateway/sds_gateway/api_methods/apps.py b/gateway/sds_gateway/api_methods/apps.py index 442f6f003..3d189bbef 100644 --- a/gateway/sds_gateway/api_methods/apps.py +++ b/gateway/sds_gateway/api_methods/apps.py @@ -12,7 +12,7 @@ class ApiMethodsConfig(AppConfig): # pattern to import application modules here in ready() # ruff: noqa: PLC0415 def ready(self) -> None: - import sds_gateway.api_methods.federation.signals # noqa: F401 + import sds_gateway.api_methods.federation.signals import sds_gateway.api_methods.schema # noqa: F401 from sds_gateway.api_methods.federation.availability import ( initialize_federation_operational_state, diff --git a/gateway/sds_gateway/api_methods/federation/availability.py b/gateway/sds_gateway/api_methods/federation/availability.py index 925ef7512..0196caf48 100644 --- a/gateway/sds_gateway/api_methods/federation/availability.py +++ b/gateway/sds_gateway/api_methods/federation/availability.py @@ -14,30 +14,32 @@ from loguru import logger as log from sds_gateway.api_methods.models import KeySources +from sds_gateway.api_methods.tasks import get_redis_client from sds_gateway.users.models import UserAPIKey +_HTTP_OK = 200 _RECHECK_INTERVAL_SECONDS = 60.0 _last_evaluated_at: float = 0.0 _cached_operational: bool = False _cached_reason: str = "not evaluated" -def _setting(name: str, default: Any = None) -> Any: +def _setting(name: str, *, default: Any = None) -> Any: return getattr(settings, name, default) def _parse_cidrs(raw: list[str]) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]: - networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] - for item in raw: - networks.append(ipaddress.ip_network(item.strip(), strict=False)) - return networks + return [ipaddress.ip_network(item.strip(), strict=False) for item in raw] def federation_client_ip(request) -> str | None: """Resolve client IP for federation export access control.""" - trust_forwarded = _setting("FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR", False) + trust_forwarded = _setting( + "FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR", + default=False, + ) if trust_forwarded: - forwarded = request.META.get("HTTP_X_FORWARDED_FOR", "") + forwarded = request.headers.get("x-forwarded-for", "") if forwarded: return forwarded.split(",")[0].strip() remote = request.META.get("REMOTE_ADDR") @@ -47,7 +49,7 @@ def federation_client_ip(request) -> str | None: def is_client_ip_allowed_for_federation_export(request) -> bool: - cidrs = _parse_cidrs(_setting("FEDERATION_EXPORT_ALLOWED_CIDRS", [])) + cidrs = _parse_cidrs(_setting("FEDERATION_EXPORT_ALLOWED_CIDRS", default=[])) if not cidrs: return False client_ip = federation_client_ip(request) @@ -61,7 +63,7 @@ def is_client_ip_allowed_for_federation_export(request) -> bool: def is_federation_internal_header_valid(request) -> bool: - secret = _setting("FEDERATION_EXPORT_INTERNAL_HEADER_SECRET", "") + secret = _setting("FEDERATION_EXPORT_INTERNAL_HEADER_SECRET", default="") if not secret: return True header_name = _setting( @@ -75,17 +77,21 @@ def is_federation_internal_header_valid(request) -> bool: return secrets.compare_digest(str(provided), str(secret)) -def _sync_health_ok() -> tuple[bool, str]: - if _setting("FEDERATION_SKIP_SYNC_HEALTH_PROBE", False): +def _sync_health_ok() -> tuple[bool, str]: # noqa: PLR0911 + if _setting("FEDERATION_SKIP_SYNC_HEALTH_PROBE", default=False): return True, "health probe skipped" url = (_setting("FEDERATION_SYNC_HEALTH_URL") or "").strip() if not url: return False, "FEDERATION_SYNC_HEALTH_URL is not set" - timeout = float(_setting("FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT", 2.0)) - request = urllib.request.Request(url, method="GET") + if not url.startswith(("http://", "https://")): + return False, "FEDERATION_SYNC_HEALTH_URL must be http(s)" + timeout = float( + _setting("FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT", default=2.0), + ) + request = urllib.request.Request(url, method="GET") # noqa: S310 try: - with urllib.request.urlopen(request, timeout=timeout) as response: - if response.status != 200: + with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 + if response.status != _HTTP_OK: return False, f"sync health returned HTTP {response.status}" body = response.read().decode("utf-8", errors="replace") except urllib.error.URLError as exc: @@ -104,7 +110,7 @@ def _sync_health_ok() -> tuple[bool, str]: def _sync_api_key_present() -> tuple[bool, str]: - if _setting("FEDERATION_SKIP_SYNC_API_KEY_CHECK", False): + if _setting("FEDERATION_SKIP_SYNC_API_KEY_CHECK", default=False): return True, "sync API key check skipped" exists = UserAPIKey.objects.filter(source=KeySources.FederationSync).exists() if not exists: @@ -113,11 +119,10 @@ def _sync_api_key_present() -> tuple[bool, str]: def _redis_ok() -> tuple[bool, str]: - if not _setting("FEDERATION_EVENTS_ENABLED", False): + if not _setting("FEDERATION_EVENTS_ENABLED", default=False): return True, "redis not required (events disabled)" - if _setting("FEDERATION_SKIP_REDIS_PROBE", False): + if _setting("FEDERATION_SKIP_REDIS_PROBE", default=False): return True, "redis probe skipped" - from sds_gateway.api_methods.tasks import get_redis_client try: client = get_redis_client() @@ -128,7 +133,7 @@ def _redis_ok() -> tuple[bool, str]: def evaluate_federation_operational() -> tuple[bool, str]: - if not _setting("FEDERATION_ENABLED", False): + if not _setting("FEDERATION_ENABLED", default=False): return False, "FEDERATION_ENABLED is False" for check in (_sync_api_key_present, _sync_health_ok, _redis_ok): @@ -139,7 +144,7 @@ def evaluate_federation_operational() -> tuple[bool, str]: def refresh_federation_operational_state(*, force: bool = False) -> tuple[bool, str]: - global _cached_operational, _cached_reason, _last_evaluated_at + global _cached_operational, _cached_reason, _last_evaluated_at # noqa: PLW0603 now = time.monotonic() if ( @@ -167,9 +172,9 @@ def initialize_federation_operational_state() -> None: def is_federation_operational() -> bool: - if _setting("FEDERATION_OPERATIONAL_OVERRIDE", None) is not None: + if _setting("FEDERATION_OPERATIONAL_OVERRIDE", default=None) is not None: return bool(_setting("FEDERATION_OPERATIONAL_OVERRIDE")) - if not _setting("FEDERATION_ENABLED", False): + if not _setting("FEDERATION_ENABLED", default=False): return False operational, _reason = refresh_federation_operational_state() return operational diff --git a/gateway/sds_gateway/api_methods/federation/events.py b/gateway/sds_gateway/api_methods/federation/events.py index 7e90af3a8..82554387f 100644 --- a/gateway/sds_gateway/api_methods/federation/events.py +++ b/gateway/sds_gateway/api_methods/federation/events.py @@ -5,16 +5,20 @@ import json from datetime import UTC from datetime import datetime +from typing import TYPE_CHECKING from typing import Any -from uuid import UUID from django.conf import settings from loguru import logger as log from sds_gateway.api_methods.federation.availability import is_federation_operational -from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.tasks import get_redis_client +if TYPE_CHECKING: + from uuid import UUID + + from sds_gateway.api_methods.models import ItemType + FederationEventType = str # created | updated | deleted diff --git a/gateway/sds_gateway/api_methods/federation/export_contract.py b/gateway/sds_gateway/api_methods/federation/export_contract.py index 7d9931f67..0da4b7f90 100644 --- a/gateway/sds_gateway/api_methods/federation/export_contract.py +++ b/gateway/sds_gateway/api_methods/federation/export_contract.py @@ -2,9 +2,11 @@ from __future__ import annotations +from typing import TYPE_CHECKING from typing import Any -from rest_framework.serializers import BaseSerializer +if TYPE_CHECKING: + from rest_framework.serializers import BaseSerializer def serializer_output_field_names(serializer: BaseSerializer[Any]) -> set[str]: @@ -22,5 +24,8 @@ def assert_field_names_match( if expected != actual: missing = expected - actual extra = actual - expected - msg = f"{label} field mismatch: missing={sorted(missing)!r} extra={sorted(extra)!r}" + msg = ( + f"{label} field mismatch: missing={sorted(missing)!r} " + f"extra={sorted(extra)!r}" + ) raise AssertionError(msg) diff --git a/gateway/sds_gateway/api_methods/federation/signals.py b/gateway/sds_gateway/api_methods/federation/signals.py index d235aae58..fa30951a5 100644 --- a/gateway/sds_gateway/api_methods/federation/signals.py +++ b/gateway/sds_gateway/api_methods/federation/signals.py @@ -25,25 +25,26 @@ def _event_type(*, created: bool, exportable: bool) -> str: return "deleted" return "created" if created else "updated" + def _skip_signal() -> bool: if not getattr(settings, "FEDERATION_EVENTS_ENABLED", False): log.debug( - "FEDERATION_EVENTS_ENABLED is False, " - "skipping federation signal", + "FEDERATION_EVENTS_ENABLED is False, skipping federation signal", ) return True - + if not is_federation_operational(): log.debug("Federation not operational, skipping signal") return True return False + @receiver(post_save, sender=Dataset) def federation_dataset_changed( sender: type[Dataset], instance: Dataset, - created: bool, + created: bool, # noqa: FBT001 **kwargs, ) -> None: if _skip_signal(): @@ -62,7 +63,7 @@ def federation_dataset_changed( def federation_capture_changed( sender: type[Capture], instance: Capture, - created: bool, + created: bool, # noqa: FBT001 **kwargs, ) -> None: if _skip_signal(): diff --git a/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py b/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py index 48db49377..365b2c201 100644 --- a/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py +++ b/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py @@ -10,22 +10,21 @@ from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset from sds_gateway.api_methods.models import DatasetStatus -from sds_gateway.api_methods.serializers.dataset_serializers import ( - get_dataset_serializer, +from sds_gateway.api_methods.serializers.capture_serializers import ( + CaptureFederationSerializer, ) from sds_gateway.api_methods.serializers.dataset_serializers import ( DatasetFederationSerializer, ) -from sds_gateway.api_methods.serializers.capture_serializers import ( - CaptureFederationSerializer, -) if TYPE_CHECKING: from django.db.models import QuerySet def federation_site_name() -> str: - return getattr(settings, "FEDERATION_SITE_NAME", settings.SDS_PROGRAMMATIC_SITE_NAME) + return getattr( + settings, "FEDERATION_SITE_NAME", settings.SDS_PROGRAMMATIC_SITE_NAME + ) def public_datasets_queryset() -> QuerySet[Dataset]: diff --git a/gateway/sds_gateway/api_methods/permissions.py b/gateway/sds_gateway/api_methods/permissions.py index 2af3b8929..2fe7bf2eb 100644 --- a/gateway/sds_gateway/api_methods/permissions.py +++ b/gateway/sds_gateway/api_methods/permissions.py @@ -26,6 +26,6 @@ class DisallowFederationSyncKey(BasePermission): def has_permission(self, request, view) -> bool: key = request.auth - if isinstance(key, UserAPIKey) and key.source == KeySources.FederationSync: - return False - return True + return not ( + isinstance(key, UserAPIKey) and key.source == KeySources.FederationSync + ) diff --git a/gateway/sds_gateway/api_methods/serializers/capture_serializers.py b/gateway/sds_gateway/api_methods/serializers/capture_serializers.py index 56156b777..168178013 100644 --- a/gateway/sds_gateway/api_methods/serializers/capture_serializers.py +++ b/gateway/sds_gateway/api_methods/serializers/capture_serializers.py @@ -12,7 +12,6 @@ from rest_framework.utils.serializer_helpers import ReturnList from sds_gateway.api_methods.helpers.index_handling import retrieve_indexed_metadata -from sds_gateway.api_methods.utils.relationship_utils import get_capture_datasets from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import CaptureType from sds_gateway.api_methods.models import DEPRECATEDPostProcessedData @@ -935,4 +934,4 @@ def get_capture_props(self, obj: Capture) -> dict[str, Any]: def get_dataset_ids(self, obj: Capture) -> list[str]: qs = get_capture_datasets(obj, include_deleted=False) - return [str(dataset.uuid) for dataset in qs] \ No newline at end of file + return [str(dataset.uuid) for dataset in qs] diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py index 0c86277d1..910e0731c 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py @@ -11,6 +11,8 @@ from sds_gateway.api_methods.federation.export_contract import assert_field_names_match from sds_gateway.api_methods.helpers.compile_federated_data import ( compile_federated_capture_doc, +) +from sds_gateway.api_methods.helpers.compile_federated_data import ( compile_federated_dataset_doc, ) from sds_gateway.api_methods.models import DatasetStatus @@ -30,10 +32,8 @@ pytest.importorskip("sds_federation") -from sds_federation.schemas.webhooks import ( # noqa: E402 - FederatedCaptureDoc, - FederatedDatasetDoc, -) +from sds_federation.schemas.webhooks import FederatedCaptureDoc # noqa: E402 +from sds_federation.schemas.webhooks import FederatedDatasetDoc # noqa: E402 User = get_user_model() diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py index bc2c97b14..7a20f8ce1 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -6,25 +6,33 @@ from unittest.mock import patch import pytest +from django.conf import settings +from django.contrib.auth import get_user_model from django.test import RequestFactory from django.test import override_settings from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase -from sds_gateway.api_methods.federation.availability import evaluate_federation_operational +from sds_gateway.api_methods.federation.availability import ( + evaluate_federation_operational, +) from sds_gateway.api_methods.federation.availability import ( is_client_ip_allowed_for_federation_export, ) from sds_gateway.api_methods.federation.availability import ( is_federation_internal_header_valid, ) -from sds_gateway.api_methods.federation.availability import refresh_federation_operational_state +from sds_gateway.api_methods.federation.availability import ( + refresh_federation_operational_state, +) from sds_gateway.api_methods.models import DatasetStatus from sds_gateway.api_methods.models import KeySources from sds_gateway.api_methods.tests.factories import DatasetFactory from sds_gateway.users.models import UserAPIKey +User = get_user_model() + pytestmark = pytest.mark.django_db @@ -42,7 +50,7 @@ def test_disabled_when_master_switch_off(self) -> None: FEDERATION_SKIP_REDIS_PROBE=True, ) def test_operational_when_probes_skipped(self) -> None: - ok, reason = evaluate_federation_operational() + ok, _reason = evaluate_federation_operational() assert ok is True @override_settings( @@ -84,7 +92,7 @@ def test_client_ip_allowlist(self) -> None: assert is_client_ip_allowed_for_federation_export(denied) is False @override_settings( - FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="top-secret", + FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="top-secret", # noqa: S106 FEDERATION_EXPORT_INTERNAL_HEADER_NAME="X-SDS-Federation-Internal", ) def test_internal_header_when_configured(self) -> None: @@ -109,7 +117,6 @@ def test_internal_header_optional_when_secret_empty(self) -> None: ) def test_refresh_sets_settings_flags(self) -> None: operational, reason = refresh_federation_operational_state(force=True) - from django.conf import settings assert operational is True assert settings.FEDERATION_OPERATIONAL is True @@ -126,9 +133,6 @@ class FederationExportAccessControlTest(APITestCase): """Export API with operational + network permissions enabled for tests.""" def setUp(self) -> None: - from django.contrib.auth import get_user_model - - User = get_user_model() self.sync_user = User.objects.create( email="sync@internal.local", is_approved=True, @@ -184,7 +188,7 @@ def test_export_returns_503_when_not_operational(self) -> None: assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE @override_settings( - FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="edge-secret", + FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="edge-secret", # noqa: S106 FEDERATION_OPERATIONAL_OVERRIDE=True, FEDERATION_EXPORT_ALLOWED_CIDRS=["127.0.0.1/32"], ) diff --git a/gateway/sds_gateway/api_methods/views/federation_endpoints.py b/gateway/sds_gateway/api_methods/views/federation_endpoints.py index 087f3472d..d9a215a26 100644 --- a/gateway/sds_gateway/api_methods/views/federation_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/federation_endpoints.py @@ -2,14 +2,22 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from django.shortcuts import get_object_or_404 from drf_spectacular.utils import extend_schema from rest_framework.decorators import action -from rest_framework.request import Request from rest_framework.response import Response from rest_framework.viewsets import ViewSet +if TYPE_CHECKING: + from rest_framework.request import Request + from sds_gateway.api_methods.authentication import APIKeyAuthentication +from sds_gateway.api_methods.federation.permissions import ( + IsFederationInternalExportClient, +) +from sds_gateway.api_methods.federation.permissions import IsFederationOperational from sds_gateway.api_methods.helpers.compile_federated_data import ( compile_federated_capture_doc, ) @@ -30,8 +38,6 @@ ) from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset -from sds_gateway.api_methods.federation.permissions import IsFederationInternalExportClient -from sds_gateway.api_methods.federation.permissions import IsFederationOperational from sds_gateway.api_methods.permissions import IsFederationSyncKey @@ -59,7 +65,9 @@ def export_datasets_list(self, request: Request) -> Response: methods=["get"], url_path=r"export/datasets/(?P[^/.]+)", ) - def export_dataset_detail(self, request: Request, pk: str | None = None) -> Response: + def export_dataset_detail( + self, request: Request, pk: str | None = None + ) -> Response: """Return one public dataset for sync after a local Redis event.""" dataset = get_object_or_404(Dataset, pk=pk, is_deleted=False) if not is_federation_exportable_dataset(dataset): @@ -82,7 +90,9 @@ def export_captures_list(self, request: Request) -> Response: methods=["get"], url_path=r"export/captures/(?P[^/.]+)", ) - def export_capture_detail(self, request: Request, pk: str | None = None) -> Response: + def export_capture_detail( + self, request: Request, pk: str | None = None + ) -> Response: """Return one public capture for sync after a local Redis event.""" capture = get_object_or_404(Capture, pk=pk, is_deleted=False) if not is_federation_exportable_capture(capture): From 3b3ea8f4d8b34270291c42806097ca975d5b41ee Mon Sep 17 00:00:00 2001 From: klpoland Date: Wed, 24 Jun 2026 16:14:00 -0400 Subject: [PATCH 04/17] arg to kwarg --- gateway/sds_gateway/api_methods/federation/availability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateway/sds_gateway/api_methods/federation/availability.py b/gateway/sds_gateway/api_methods/federation/availability.py index 0196caf48..718bf10a1 100644 --- a/gateway/sds_gateway/api_methods/federation/availability.py +++ b/gateway/sds_gateway/api_methods/federation/availability.py @@ -68,7 +68,7 @@ def is_federation_internal_header_valid(request) -> bool: return True header_name = _setting( "FEDERATION_EXPORT_INTERNAL_HEADER_NAME", - "X-SDS-Federation-Internal", + default="X-SDS-Federation-Internal", ) meta_key = "HTTP_" + header_name.upper().replace("-", "_") provided = request.META.get(meta_key, "") From 4bbcbed360f2aca9de6108ccd0b94b400b730ff0 Mon Sep 17 00:00:00 2001 From: klpoland Date: Thu, 25 Jun 2026 11:20:57 -0400 Subject: [PATCH 05/17] fix test errors --- gateway/config/settings/base.py | 2 +- .../users/management/commands/create_test_files.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index 20ceef92b..71455b416 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -714,7 +714,7 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: FEDERATION_SITE_NAME: str = env.str( "FEDERATION_SITE_NAME", - default="", + default="crc", ) # Master switch: when False, federation export and Redis events are inactive. FEDERATION_ENABLED: bool = env.bool("FEDERATION_ENABLED", default=False) diff --git a/gateway/sds_gateway/users/management/commands/create_test_files.py b/gateway/sds_gateway/users/management/commands/create_test_files.py index 9c6f0b553..b85b28e0d 100644 --- a/gateway/sds_gateway/users/management/commands/create_test_files.py +++ b/gateway/sds_gateway/users/management/commands/create_test_files.py @@ -10,9 +10,9 @@ from django.db import transaction from django.utils import timezone -from sds_gateway.captures.models import Capture -from sds_gateway.files.models import File -from sds_gateway.minio_client import MinioClient +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import File +from sds_gateway.api_methods.utils.minio_client import get_minio_client User = get_user_model() # This is the only User import we need @@ -69,7 +69,7 @@ def _create_test_files(self, user): directory_path.mkdir(parents=True, exist_ok=True) # Create files - minio_client = MinioClient() + minio_client = get_minio_client() created_files = [] try: From 2c09faef79e62eeb1de9193ac4c099f987d2252c Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 26 Jun 2026 11:56:33 -0400 Subject: [PATCH 06/17] clean up env variable settings, ip parsing, auth classes --- gateway/.envs/example/django.env | 22 ++++---- gateway/config/settings/base.py | 53 ++++++++---------- .../api_methods/federation/availability.py | 48 +++++++---------- .../api_methods/federation/events.py | 8 ++- .../api_methods/federation/export_contract.py | 3 +- .../api_methods/federation/permissions.py | 9 +--- .../api_methods/federation/signals.py | 6 +-- .../helpers/compile_federated_data.py | 4 +- .../tests/test_federation_export.py | 4 +- .../tests/test_federation_hardening.py | 54 +++++-------------- .../api_methods/views/dataset_endpoints.py | 2 +- 11 files changed, 82 insertions(+), 131 deletions(-) diff --git a/gateway/.envs/example/django.env b/gateway/.envs/example/django.env index 356c45345..f306a3f6b 100644 --- a/gateway/.envs/example/django.env +++ b/gateway/.envs/example/django.env @@ -10,18 +10,16 @@ API_KEY= # FEDERATION # ------------------------------------------------------------------------------ -# the below environment variables are used for site federation with peers. -# By default, federation events are disabled. -# If FEDERATION_ENABLED is TRUE, deployment will need to include -# federation sync service configuration. -# FEDERATION_ENABLED=true -# FEDERATION_EVENTS_ENABLED=true -# FEDERATION_EVENTS_CHANNEL=federation:events -# FEDERATION_SYNC_USER_EMAIL= -# FEDERATION_SITE_NAME= -# FEDERATION_SYNC_HEALTH_URL=http://federation-sync:8000/sync/health -# FEDERATION_EXPORT_INTERNAL_HEADER_SECRET= -# FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR=false +# Peer sync uses the federation-sync Docker service (same sds-network as gateway; +# service definition lives under /federation). Bootstrap: enable federation, run +# create_federation_sync_api_key, pass the key to federation-sync. Set FEDERATION_SITE_NAME +# (e.g. crc) when enabling federation; use SDS_SITE_FQDN for the public host (RFC [site].fqdn). +# FEDERATION_ENABLED=true # Master switch for export APIs and Redis federation events. +# FEDERATION_SITE_NAME=crc # RFC [site].name (short peer id); set SDS_SITE_FQDN separately for [site].fqdn. +# FEDERATION_EVENTS_CHANNEL=federation:events # Redis pub/sub channel federation-sync subscribes to. +# FEDERATION_SYNC_HEALTH_URL=http://federation-sync:8000/sync/health # Health probe target (federation-sync service). +# FEDERATION_SYNC_USER_EMAIL=federation-sync@internal.local # Service user email for create_federation_sync_api_key. +# FEDERATION_EXPORT_ALLOWED_CIDRS= # Comma-separated CIDRs allowed to call export (default: private Docker ranges). # AUTH0 # ------------------------------------------------------------------------------ diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index 71455b416..e254b1f61 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -1,6 +1,7 @@ """Base settings to build other settings files upon.""" # ruff: noqa: ERA001 +import ipaddress import random import string from pathlib import Path @@ -23,6 +24,8 @@ def __get_random_token(length: int) -> str: __rng.choice(string.ascii_letters + string.digits) for _ in range(length) ) +def _parse_cidrs(raw: list[str]) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]: + return [ipaddress.ip_network(item.strip(), strict=False) for item in raw] env.read_env() @@ -712,13 +715,10 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: SDS_PROGRAMMATIC_SITE_NAME: str = env.str("SDS_PROGRAMMATIC_SITE_NAME", default="sds") SDS_SITE_FQDN: str = env.str("SDS_SITE_FQDN", default="localhost") -FEDERATION_SITE_NAME: str = env.str( - "FEDERATION_SITE_NAME", - default="crc", -) +# Federation peer short name (RFC [site].name, e.g. crc, haystack); not SDS_PROGRAMMATIC_SITE_NAME. +FEDERATION_SITE_NAME: str = env.str("FEDERATION_SITE_NAME", default="").strip() # Master switch: when False, federation export and Redis events are inactive. FEDERATION_ENABLED: bool = env.bool("FEDERATION_ENABLED", default=False) -FEDERATION_EVENTS_ENABLED: bool = env.bool("FEDERATION_EVENTS_ENABLED", default=False) FEDERATION_EVENTS_CHANNEL: str = env.str( "FEDERATION_EVENTS_CHANNEL", default="federation:events", @@ -727,8 +727,10 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: "FEDERATION_SYNC_USER_EMAIL", default="federation-sync@internal.local", ) -# Sync service health (e.g. http://federation-sync:8000/sync/health). -FEDERATION_SYNC_HEALTH_URL: str = env.str("FEDERATION_SYNC_HEALTH_URL", default="") +FEDERATION_SYNC_HEALTH_URL: str = env.str( + "FEDERATION_SYNC_HEALTH_URL", + default="http://federation-sync:8000/sync/health", +) FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT: float = env.float( "FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT", default=2.0, @@ -750,29 +752,20 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: FEDERATION_OPERATIONAL_REASON: str = "" # Tests may set via override_settings without running probes. FEDERATION_OPERATIONAL_OVERRIDE: bool | None = None -# Export API: allow only these source CIDRs (sync container / internal mesh). -FEDERATION_EXPORT_ALLOWED_CIDRS: list[str] = env.list( - "FEDERATION_EXPORT_ALLOWED_CIDRS", - default=[ - "127.0.0.1/32", - "::1/128", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - ], -) -FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR: bool = env.bool( - "FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR", - default=False, -) -# Optional: Traefik injects this on internal routes only (empty = IP check only). -FEDERATION_EXPORT_INTERNAL_HEADER_NAME: str = env.str( - "FEDERATION_EXPORT_INTERNAL_HEADER_NAME", - default="X-SDS-Federation-Internal", -) -FEDERATION_EXPORT_INTERNAL_HEADER_SECRET: str = env.str( - "FEDERATION_EXPORT_INTERNAL_HEADER_SECRET", - default="", +# Export API: internal Docker/private networks (sync → django on sds-network). +FEDERATION_EXPORT_ALLOWED_CIDRS: list[ + ipaddress.IPv4Network | ipaddress.IPv6Network +] = _parse_cidrs( + env.list( + "FEDERATION_EXPORT_ALLOWED_CIDRS", + default=[ + "127.0.0.1/32", + "::1/128", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + ], + ), ) # ADMIN_CONSOLE_ENV is used to visually distinguish between different environments diff --git a/gateway/sds_gateway/api_methods/federation/availability.py b/gateway/sds_gateway/api_methods/federation/availability.py index 718bf10a1..f497c0758 100644 --- a/gateway/sds_gateway/api_methods/federation/availability.py +++ b/gateway/sds_gateway/api_methods/federation/availability.py @@ -4,7 +4,6 @@ import ipaddress import json -import secrets import time import urllib.error import urllib.request @@ -28,20 +27,20 @@ def _setting(name: str, *, default: Any = None) -> Any: return getattr(settings, name, default) -def _parse_cidrs(raw: list[str]) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]: - return [ipaddress.ip_network(item.strip(), strict=False) for item in raw] +def _export_allowed_networks() -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]: + """Networks from settings (parsed at startup); tests may override with strings.""" + raw = _setting("FEDERATION_EXPORT_ALLOWED_CIDRS", default=[]) + networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] + for item in raw: + if isinstance(item, (ipaddress.IPv4Network, ipaddress.IPv6Network)): + networks.append(item) + else: + networks.append(ipaddress.ip_network(str(item).strip(), strict=False)) + return networks def federation_client_ip(request) -> str | None: - """Resolve client IP for federation export access control.""" - trust_forwarded = _setting( - "FEDERATION_EXPORT_TRUST_X_FORWARDED_FOR", - default=False, - ) - if trust_forwarded: - forwarded = request.headers.get("x-forwarded-for", "") - if forwarded: - return forwarded.split(",")[0].strip() + """Client IP for export access control (direct internal connections only).""" remote = request.META.get("REMOTE_ADDR") if remote: return str(remote).strip() @@ -49,7 +48,7 @@ def federation_client_ip(request) -> str | None: def is_client_ip_allowed_for_federation_export(request) -> bool: - cidrs = _parse_cidrs(_setting("FEDERATION_EXPORT_ALLOWED_CIDRS", default=[])) + cidrs = _export_allowed_networks() if not cidrs: return False client_ip = federation_client_ip(request) @@ -62,21 +61,6 @@ def is_client_ip_allowed_for_federation_export(request) -> bool: return any(addr in network for network in cidrs) -def is_federation_internal_header_valid(request) -> bool: - secret = _setting("FEDERATION_EXPORT_INTERNAL_HEADER_SECRET", default="") - if not secret: - return True - header_name = _setting( - "FEDERATION_EXPORT_INTERNAL_HEADER_NAME", - default="X-SDS-Federation-Internal", - ) - meta_key = "HTTP_" + header_name.upper().replace("-", "_") - provided = request.META.get(meta_key, "") - if not provided: - return False - return secrets.compare_digest(str(provided), str(secret)) - - def _sync_health_ok() -> tuple[bool, str]: # noqa: PLR0911 if _setting("FEDERATION_SKIP_SYNC_HEALTH_PROBE", default=False): return True, "health probe skipped" @@ -119,8 +103,8 @@ def _sync_api_key_present() -> tuple[bool, str]: def _redis_ok() -> tuple[bool, str]: - if not _setting("FEDERATION_EVENTS_ENABLED", default=False): - return True, "redis not required (events disabled)" + if not _setting("FEDERATION_ENABLED", default=False): + return True, "redis not required (federation disabled)" if _setting("FEDERATION_SKIP_REDIS_PROBE", default=False): return True, "redis probe skipped" @@ -136,6 +120,10 @@ def evaluate_federation_operational() -> tuple[bool, str]: if not _setting("FEDERATION_ENABLED", default=False): return False, "FEDERATION_ENABLED is False" + site_name = (_setting("FEDERATION_SITE_NAME", default="") or "").strip() + if not site_name: + return False, "FEDERATION_SITE_NAME must be set when federation is enabled" + for check in (_sync_api_key_present, _sync_health_ok, _redis_ok): ok, reason = check() if not ok: diff --git a/gateway/sds_gateway/api_methods/federation/events.py b/gateway/sds_gateway/api_methods/federation/events.py index 82554387f..1fc815847 100644 --- a/gateway/sds_gateway/api_methods/federation/events.py +++ b/gateway/sds_gateway/api_methods/federation/events.py @@ -19,7 +19,13 @@ from sds_gateway.api_methods.models import ItemType -FederationEventType = str # created | updated | deleted +class FederationEventType(StrEnum): + CREATED = "created" + UPDATED = "updated" + DELETED = "deleted" + + def __str__(self) -> str: + return self.value def publish_federation_event( diff --git a/gateway/sds_gateway/api_methods/federation/export_contract.py b/gateway/sds_gateway/api_methods/federation/export_contract.py index 0da4b7f90..0a7d65df1 100644 --- a/gateway/sds_gateway/api_methods/federation/export_contract.py +++ b/gateway/sds_gateway/api_methods/federation/export_contract.py @@ -6,6 +6,7 @@ from typing import Any if TYPE_CHECKING: + from pydantic import BaseModel from rest_framework.serializers import BaseSerializer @@ -15,7 +16,7 @@ def serializer_output_field_names(serializer: BaseSerializer[Any]) -> set[str]: def assert_field_names_match( serializer: BaseSerializer[Any], - pydantic_model: type, + pydantic_model: BaseModel, *, label: str, ) -> None: diff --git a/gateway/sds_gateway/api_methods/federation/permissions.py b/gateway/sds_gateway/api_methods/federation/permissions.py index 61f2ecad8..2936c2e62 100644 --- a/gateway/sds_gateway/api_methods/federation/permissions.py +++ b/gateway/sds_gateway/api_methods/federation/permissions.py @@ -7,9 +7,6 @@ from sds_gateway.api_methods.federation.availability import ( is_client_ip_allowed_for_federation_export, ) -from sds_gateway.api_methods.federation.availability import ( - is_federation_internal_header_valid, -) from sds_gateway.api_methods.federation.availability import is_federation_operational @@ -34,11 +31,9 @@ def has_permission(self, request, view) -> bool: class IsFederationInternalExportClient(BasePermission): - """Restrict export to internal networks (and optional edge-injected header).""" + """Restrict export to internal network clients (sync service on sds-network).""" message = "Federation export is only available to internal clients." def has_permission(self, request, view) -> bool: - if not is_client_ip_allowed_for_federation_export(request): - return False - return is_federation_internal_header_valid(request) + return is_client_ip_allowed_for_federation_export(request) diff --git a/gateway/sds_gateway/api_methods/federation/signals.py b/gateway/sds_gateway/api_methods/federation/signals.py index fa30951a5..c5cc06e04 100644 --- a/gateway/sds_gateway/api_methods/federation/signals.py +++ b/gateway/sds_gateway/api_methods/federation/signals.py @@ -27,10 +27,8 @@ def _event_type(*, created: bool, exportable: bool) -> str: def _skip_signal() -> bool: - if not getattr(settings, "FEDERATION_EVENTS_ENABLED", False): - log.debug( - "FEDERATION_EVENTS_ENABLED is False, skipping federation signal", - ) + if not getattr(settings, "FEDERATION_ENABLED", False): + log.debug("FEDERATION_ENABLED is False, skipping federation signal") return True if not is_federation_operational(): diff --git a/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py b/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py index 365b2c201..53be29f53 100644 --- a/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py +++ b/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py @@ -22,9 +22,7 @@ def federation_site_name() -> str: - return getattr( - settings, "FEDERATION_SITE_NAME", settings.SDS_PROGRAMMATIC_SITE_NAME - ) + return getattr(settings, "FEDERATION_SITE_NAME", "").strip() def public_datasets_queryset() -> QuerySet[Dataset]: diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export.py b/gateway/sds_gateway/api_methods/tests/test_federation_export.py index 7d68dae5f..9dfe2a864 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export.py @@ -17,9 +17,9 @@ @override_settings( FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", FEDERATION_OPERATIONAL_OVERRIDE=True, FEDERATION_EXPORT_ALLOWED_CIDRS=["0.0.0.0/0", "::/0"], - FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="", ) class FederationExportAPITest(APITestCase): def setUp(self) -> None: @@ -84,7 +84,7 @@ def test_sync_key_can_retrieve_public_dataset(self) -> None: ) assert response.status_code == status.HTTP_200_OK assert response.json()["uuid"] == str(self.public_dataset.uuid) - assert response.json()["site_name"] + assert response.json()["site_name"] == "crc" def test_regular_key_denied_on_export(self) -> None: response = self.client.get( diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py index 7a20f8ce1..0fc2cfe0e 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -20,9 +20,6 @@ from sds_gateway.api_methods.federation.availability import ( is_client_ip_allowed_for_federation_export, ) -from sds_gateway.api_methods.federation.availability import ( - is_federation_internal_header_valid, -) from sds_gateway.api_methods.federation.availability import ( refresh_federation_operational_state, ) @@ -45,6 +42,7 @@ def test_disabled_when_master_switch_off(self) -> None: @override_settings( FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, @@ -55,6 +53,7 @@ def test_operational_when_probes_skipped(self) -> None: @override_settings( FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, ) @@ -65,6 +64,7 @@ def test_fails_without_sync_api_key(self) -> None: @override_settings( FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, FEDERATION_SYNC_HEALTH_URL="http://sync.test/health", FEDERATION_SKIP_REDIS_PROBE=True, @@ -92,25 +92,20 @@ def test_client_ip_allowlist(self) -> None: assert is_client_ip_allowed_for_federation_export(denied) is False @override_settings( - FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="top-secret", # noqa: S106 - FEDERATION_EXPORT_INTERNAL_HEADER_NAME="X-SDS-Federation-Internal", + FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="", + FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, + FEDERATION_SKIP_REDIS_PROBE=True, ) - def test_internal_header_when_configured(self) -> None: - factory = RequestFactory() - missing = factory.get("/") - valid = factory.get("/", HTTP_X_SDS_FEDERATION_INTERNAL="top-secret") - invalid = factory.get("/", HTTP_X_SDS_FEDERATION_INTERNAL="wrong") - assert is_federation_internal_header_valid(missing) is False - assert is_federation_internal_header_valid(valid) is True - assert is_federation_internal_header_valid(invalid) is False - - @override_settings(FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="") - def test_internal_header_optional_when_secret_empty(self) -> None: - request = RequestFactory().get("/") - assert is_federation_internal_header_valid(request) is True + def test_fails_without_site_name(self) -> None: + ok, reason = evaluate_federation_operational() + assert ok is False + assert "FEDERATION_SITE_NAME" in reason @override_settings( FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, FEDERATION_SKIP_SYNC_HEALTH_PROBE=True, FEDERATION_SKIP_REDIS_PROBE=True, @@ -125,9 +120,9 @@ def test_refresh_sets_settings_flags(self) -> None: @override_settings( FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", FEDERATION_OPERATIONAL_OVERRIDE=True, FEDERATION_EXPORT_ALLOWED_CIDRS=["0.0.0.0/0", "::/0"], - FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="", ) class FederationExportAccessControlTest(APITestCase): """Export API with operational + network permissions enabled for tests.""" @@ -186,24 +181,3 @@ def test_export_returns_503_when_not_operational(self) -> None: **self._auth(self.sync_key), ) assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE - - @override_settings( - FEDERATION_EXPORT_INTERNAL_HEADER_SECRET="edge-secret", # noqa: S106 - FEDERATION_OPERATIONAL_OVERRIDE=True, - FEDERATION_EXPORT_ALLOWED_CIDRS=["127.0.0.1/32"], - ) - def test_internal_header_required_when_configured(self) -> None: - denied = self.client.get( - self.list_datasets_url, - REMOTE_ADDR="127.0.0.1", - **self._auth(self.sync_key), - ) - assert denied.status_code == status.HTTP_403_FORBIDDEN - - allowed = self.client.get( - self.list_datasets_url, - REMOTE_ADDR="127.0.0.1", - HTTP_X_SDS_FEDERATION_INTERNAL="edge-secret", - **self._auth(self.sync_key), - ) - assert allowed.status_code == status.HTTP_200_OK diff --git a/gateway/sds_gateway/api_methods/views/dataset_endpoints.py b/gateway/sds_gateway/api_methods/views/dataset_endpoints.py index 6125fa0f0..2ef78566f 100644 --- a/gateway/sds_gateway/api_methods/views/dataset_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/dataset_endpoints.py @@ -55,7 +55,7 @@ def _truthy_query_param(raw: str | None) -> bool: class DatasetViewSet(ViewSet): - authentication_classes = [APIKeyAuthentication] + authentication_classes = [APIKeyAuthentication, SessionAuthentication] def _get_file_objects( self, From 3393c0f72bdb2e2894b75630b165684fe87cd924 Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 26 Jun 2026 12:10:11 -0400 Subject: [PATCH 07/17] gateway mirror for health check --- .../api_methods/federation/availability.py | 20 ++++++++++------ .../tests/test_federation_hardening.py | 23 +++++++++++++++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/gateway/sds_gateway/api_methods/federation/availability.py b/gateway/sds_gateway/api_methods/federation/availability.py index f497c0758..75cd6fd39 100644 --- a/gateway/sds_gateway/api_methods/federation/availability.py +++ b/gateway/sds_gateway/api_methods/federation/availability.py @@ -82,14 +82,20 @@ def _sync_health_ok() -> tuple[bool, str]: # noqa: PLR0911 return False, f"sync health probe failed: {exc.reason}" except TimeoutError: return False, "sync health probe timed out" - if body: - try: - payload = json.loads(body) - if isinstance(payload, dict) and payload.get("status") == "ok": - return True, "sync health ok" - except json.JSONDecodeError: - pass + if not body.strip(): return True, "sync health returned 200" + + try: + payload = json.loads(body) + except json.JSONDecodeError: + return True, "sync health returned 200" + + if isinstance(payload, dict): + if payload.get("status") == "ok": + return True, "sync health ok" + status_value = payload.get("status") + return False, f"sync health status is not ok: {status_value!r}" + return True, "sync health returned 200" diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py index 0fc2cfe0e..d52ef7b49 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -81,6 +81,29 @@ def test_health_probe_success(self, mock_urlopen: MagicMock) -> None: ok, _reason = evaluate_federation_operational() assert ok is True + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", + FEDERATION_SKIP_SYNC_API_KEY_CHECK=True, + FEDERATION_SYNC_HEALTH_URL="http://sync.test/health", + FEDERATION_SKIP_REDIS_PROBE=True, + ) + @patch("sds_gateway.api_methods.federation.availability.urllib.request.urlopen") + def test_health_probe_fails_when_json_status_not_ok( + self, + mock_urlopen: MagicMock, + ) -> None: + response = MagicMock() + response.status = 200 + response.read.return_value = b'{"status":"degraded"}' + response.__enter__.return_value = response + response.__exit__.return_value = None + mock_urlopen.return_value = response + + ok, reason = evaluate_federation_operational() + assert ok is False + assert "not ok" in reason + @override_settings( FEDERATION_EXPORT_ALLOWED_CIDRS=["10.0.0.0/8"], ) From 751d03aff6f8bb57481b7bdd5d593d1262bade2f Mon Sep 17 00:00:00 2001 From: klpoland Date: Thu, 2 Jul 2026 14:22:53 -0400 Subject: [PATCH 08/17] follow rfc 12: site-prefixed redis channels --- gateway/config/settings/base.py | 11 ++- .../api_methods/federation/events.py | 5 +- .../api_methods/federation/redis_channel.py | 22 +++++ .../tests/test_federation_events.py | 82 +++++++++++++++++++ 4 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 gateway/sds_gateway/api_methods/federation/redis_channel.py create mode 100644 gateway/sds_gateway/api_methods/tests/test_federation_events.py diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index e254b1f61..d176814f0 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -14,6 +14,9 @@ from config.settings.logs import ColoredFormatter from config.settings.utils import guess_admin_console_env from config.settings.utils import guess_max_web_download_size +from sds_gateway.api_methods.federation.redis_channel import ( + resolve_federation_events_channel, +) __rng = random.SystemRandom() @@ -719,9 +722,13 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: FEDERATION_SITE_NAME: str = env.str("FEDERATION_SITE_NAME", default="").strip() # Master switch: when False, federation export and Redis events are inactive. FEDERATION_ENABLED: bool = env.bool("FEDERATION_ENABLED", default=False) -FEDERATION_EVENTS_CHANNEL: str = env.str( +_federation_events_channel_override: str = env.str( "FEDERATION_EVENTS_CHANNEL", - default="federation:events", + default="", +).strip() +FEDERATION_EVENTS_CHANNEL: str = resolve_federation_events_channel( + site_name=FEDERATION_SITE_NAME, + channel_override=_federation_events_channel_override, ) FEDERATION_SYNC_USER_EMAIL: str = env.str( "FEDERATION_SYNC_USER_EMAIL", diff --git a/gateway/sds_gateway/api_methods/federation/events.py b/gateway/sds_gateway/api_methods/federation/events.py index 1fc815847..5f17e2033 100644 --- a/gateway/sds_gateway/api_methods/federation/events.py +++ b/gateway/sds_gateway/api_methods/federation/events.py @@ -5,6 +5,7 @@ import json from datetime import UTC from datetime import datetime +from enum import StrEnum from typing import TYPE_CHECKING from typing import Any @@ -39,9 +40,9 @@ def publish_federation_event( if not is_federation_operational(): log.debug("Federation not operational, skipping Redis publish") return - channel = getattr(settings, "FEDERATION_EVENTS_CHANNEL", "federation:events") + channel = settings.FEDERATION_EVENTS_CHANNEL payload: dict[str, Any] = { - "event_type": event_type, + "event_type": str(event_type), "item_type": item_type.value, "uuid": str(uuid), "timestamp": (timestamp or datetime.now(UTC)).isoformat(), diff --git a/gateway/sds_gateway/api_methods/federation/redis_channel.py b/gateway/sds_gateway/api_methods/federation/redis_channel.py new file mode 100644 index 000000000..48abf17c0 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/redis_channel.py @@ -0,0 +1,22 @@ +"""Redis pub/sub channel naming for federation change events (RFC §8).""" + +from __future__ import annotations + + +def resolve_federation_events_channel( + *, + site_name: str = "", + channel_override: str = "", +) -> str: + """Return the Redis channel for local federation events. + + Override ``channel_override`` when set (``FEDERATION_EVENTS_CHANNEL`` env). + Otherwise use ``federation:events:{site_name}`` when ``site_name`` is set. + """ + override = (channel_override or "").strip() + if override: + return override + site = (site_name or "").strip() + if site: + return f"federation:events:{site}" + return "" diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_events.py b/gateway/sds_gateway/api_methods/tests/test_federation_events.py new file mode 100644 index 000000000..62f458105 --- /dev/null +++ b/gateway/sds_gateway/api_methods/tests/test_federation_events.py @@ -0,0 +1,82 @@ +"""Tests for federation Redis event channel naming and publish.""" + +from __future__ import annotations + +from unittest.mock import MagicMock +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from django.test import override_settings + +from sds_gateway.api_methods.federation.events import FederationEventType +from sds_gateway.api_methods.federation.events import publish_federation_event +from sds_gateway.api_methods.federation.redis_channel import ( + resolve_federation_events_channel, +) +from sds_gateway.api_methods.models import ItemType + +pytestmark = pytest.mark.django_db + + +class TestResolveFederationEventsChannel: + def test_site_prefixed_when_site_name_set(self) -> None: + assert ( + resolve_federation_events_channel(site_name="crc") + == "federation:events:crc" + ) + + def test_override_wins_over_site_name(self) -> None: + assert ( + resolve_federation_events_channel( + site_name="crc", + channel_override="federation:events:custom", + ) + == "federation:events:custom" + ) + + def test_empty_when_no_site_and_no_override(self) -> None: + assert resolve_federation_events_channel() == "" + + +class TestPublishFederationEvent: + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_OPERATIONAL_OVERRIDE=True, + FEDERATION_EVENTS_CHANNEL="federation:events:crc", + ) + @patch("sds_gateway.api_methods.federation.events.get_redis_client") + def test_publish_uses_configured_channel( + self, + mock_get_redis: MagicMock, + ) -> None: + mock_client = MagicMock() + mock_get_redis.return_value = mock_client + item_uuid = uuid4() + + publish_federation_event( + event_type=FederationEventType.CREATED, + item_type=ItemType.DATASET, + uuid=item_uuid, + ) + + mock_client.publish.assert_called_once() + channel, _payload = mock_client.publish.call_args[0] + assert channel == "federation:events:crc" + + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_OPERATIONAL_OVERRIDE=False, + FEDERATION_EVENTS_CHANNEL="federation:events:crc", + ) + @patch("sds_gateway.api_methods.federation.events.get_redis_client") + def test_skips_publish_when_not_operational( + self, + mock_get_redis: MagicMock, + ) -> None: + publish_federation_event( + event_type=FederationEventType.UPDATED, + item_type=ItemType.CAPTURE, + uuid=uuid4(), + ) + mock_get_redis.assert_not_called() From 6888bc88659dc8bb720c329dad11945f9316c545 Mon Sep 17 00:00:00 2001 From: klpoland Date: Thu, 9 Jul 2026 12:09:02 -0400 Subject: [PATCH 09/17] refactor: harden publishing reversion protection, federation event logic and local indexing before notify --- .../federation/compile_federated_data.py | 160 +++++++++++ .../api_methods/federation/fed_index.py | 74 ++++++ .../api_methods/federation/signals.py | 221 +++++++++++++--- .../helpers/compile_federated_data.py | 75 ------ gateway/sds_gateway/api_methods/models.py | 42 +++ .../test_dataset_publish_immutability.py | 45 ++++ .../tests/test_federation_events.py | 43 +++ .../tests/test_federation_export.py | 34 ++- .../tests/test_federation_export_contract.py | 4 +- .../tests/test_federation_signals.py | 248 ++++++++++++++++++ .../api_methods/views/federation_endpoints.py | 74 +++--- 11 files changed, 872 insertions(+), 148 deletions(-) create mode 100644 gateway/sds_gateway/api_methods/federation/compile_federated_data.py create mode 100644 gateway/sds_gateway/api_methods/federation/fed_index.py delete mode 100644 gateway/sds_gateway/api_methods/helpers/compile_federated_data.py create mode 100644 gateway/sds_gateway/api_methods/tests/test_dataset_publish_immutability.py create mode 100644 gateway/sds_gateway/api_methods/tests/test_federation_signals.py diff --git a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py new file mode 100644 index 000000000..7e8fe166a --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py @@ -0,0 +1,160 @@ +"""Build federation export payloads and fed-* OpenSearch helpers.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any +from uuid import UUID + +from django.conf import settings +from opensearchpy.exceptions import NotFoundError + +from sds_gateway.api_methods.federation.fed_index import index_for_item_type +from sds_gateway.api_methods.federation.fed_index import federated_doc_id +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.models import ItemType +from sds_gateway.api_methods.serializers.capture_serializers import ( + CaptureFederationSerializer, +) +from sds_gateway.api_methods.serializers.dataset_serializers import ( + DatasetFederationSerializer, +) +from sds_gateway.api_methods.utils.opensearch_client import get_opensearch_client + +if TYPE_CHECKING: + from django.db.models import QuerySet + + +def federation_site_name() -> str: + return getattr(settings, "FEDERATION_SITE_NAME", "").strip() + + +def is_federation_exportable_dataset(dataset: Dataset) -> bool: + return ( + not dataset.is_deleted + and dataset.is_public + and dataset.status == DatasetStatus.FINAL + ) + + +def capture_in_published_dataset(capture: Capture) -> bool: + return capture.datasets.filter( + status=DatasetStatus.FINAL, + is_public=True, + is_deleted=False, + ).exists() + + +def capture_in_other_published_datasets( + capture: Capture, + *, + exclude_dataset_id: UUID, +) -> bool: + return ( + capture.datasets.filter( + status=DatasetStatus.FINAL, + is_public=True, + is_deleted=False, + ) + .exclude(uuid=exclude_dataset_id) + .exists() + ) + + +def compile_federated_dataset_doc(dataset: Dataset) -> dict[str, Any]: + site_name = federation_site_name() + return DatasetFederationSerializer( + dataset, + context={"site_name": site_name}, + ).data + + +def compile_federated_capture_doc(capture: Capture) -> dict[str, Any]: + site_name = federation_site_name() + return CaptureFederationSerializer( + capture, + context={"site_name": site_name}, + ).data + + +def compile_federated_doc( + instance: Dataset | Capture, +) -> dict[str, Any]: + if isinstance(instance, Dataset): + return compile_federated_dataset_doc(instance) + return compile_federated_capture_doc(instance) + + +def get_federated_export_doc_by_uuid( + *, + asset_type: ItemType, + asset_uuid: UUID, + site_name: str | None = None, +) -> dict[str, Any] | None: + """Return indexed federation export body for this site, or None if missing.""" + resolved_site = (site_name or federation_site_name()).strip() + if not resolved_site: + return None + client = get_opensearch_client() + index = index_for_item_type(asset_type) + doc_id = federated_doc_id( + site_name=resolved_site, + uuid=asset_uuid, + ) + try: + response = client.get(index=index, id=doc_id) + except NotFoundError: + return None + source = response.get("_source") + if not isinstance(source, dict): + return None + return source + + +def fed_doc_exists( + *, + asset_type: ItemType, + site_name: str, + instance: Dataset | Capture, +) -> bool: + return get_federated_export_doc_by_uuid( + asset_type=asset_type, + asset_uuid=instance.uuid, + site_name=site_name, + ) is not None + + +def fed_doc_is_tombstoned( + *, + asset_type: ItemType, + site_name: str, + instance: Dataset | Capture, +) -> bool: + source = get_federated_export_doc_by_uuid( + asset_type=asset_type, + asset_uuid=instance.uuid, + site_name=site_name, + ) + if source is None: + return False + return source.get("is_federated_deleted") is True + + +def public_datasets_queryset() -> QuerySet[Dataset]: + return ( + Dataset.objects.filter( + status=DatasetStatus.FINAL, + is_public=True, + is_deleted=False, + ) + .prefetch_related("keywords", "owner") + .order_by("-updated_at") + ) + + +def public_captures_queryset() -> QuerySet[Capture]: + return Capture.objects.filter(is_public=True, is_deleted=False).order_by( + "-updated_at", + ) diff --git a/gateway/sds_gateway/api_methods/federation/fed_index.py b/gateway/sds_gateway/api_methods/federation/fed_index.py new file mode 100644 index 000000000..b04fc9e95 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/fed_index.py @@ -0,0 +1,74 @@ +"""Write local site documents into fed-* OpenSearch indices.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any +from uuid import UUID + +from opensearchpy import OpenSearch +from sds_gateway.api_methods.models import ItemType + +_EVENT_DELETED = "deleted" +FED_DATASETS_INDEX = "fed-datasets" +FED_CAPTURES_INDEX = "fed-captures" + + +def federated_doc_id(site_name: str, uuid: UUID) -> str: + return f"{site_name}:{uuid}" + + +def index_for_item_type(item_type: ItemType) -> str: + if item_type == ItemType.DATASET: + return FED_DATASETS_INDEX + if item_type == ItemType.CAPTURE: + return FED_CAPTURES_INDEX + msg = f"unsupported federation item type: {item_type}" + raise ValueError(msg) + + +class LocalFederatedIndexer: + """Index this site's public federation export shape into shared fed-* indices.""" + + def __init__(self, client: OpenSearch) -> None: + self._client = client + + def apply_local_event( + self, + *, + event_type: str, + event_at: datetime, + site_name: str, + item_type: ItemType, + uuid: UUID, + body: dict[str, Any], + ) -> None: + index_name = index_for_item_type(item_type) + doc_id = federated_doc_id(site_name, uuid) + event_iso = event_at.isoformat() + + if event_type == _EVENT_DELETED: + tombstone = { + "is_federated_deleted": True, + "federation_event_at": event_iso, + } + upsert_body = {**body, **tombstone} + self._client.update( + index=index_name, + id=doc_id, + body={"doc": tombstone, "upsert": upsert_body}, + refresh="wait_for", + ) + return + + doc = { + **body, + "is_federated_deleted": False, + "federation_event_at": event_iso, + } + self._client.index( + index=index_name, + id=doc_id, + body=doc, + refresh="wait_for", + ) diff --git a/gateway/sds_gateway/api_methods/federation/signals.py b/gateway/sds_gateway/api_methods/federation/signals.py index c5cc06e04..9b47aa809 100644 --- a/gateway/sds_gateway/api_methods/federation/signals.py +++ b/gateway/sds_gateway/api_methods/federation/signals.py @@ -1,4 +1,4 @@ -"""Django signals that publish federation Redis events.""" +"""Index local fed-* docs on save, then notify sync via Redis.""" from __future__ import annotations @@ -7,35 +7,182 @@ from django.dispatch import receiver from loguru import logger as log -from sds_gateway.api_methods.federation.availability import is_federation_operational -from sds_gateway.api_methods.federation.events import publish_federation_event -from sds_gateway.api_methods.helpers.compile_federated_data import ( - is_federation_exportable_capture, +from sds_gateway.api_methods.federation.compile_federated_data import ( + capture_in_other_published_datasets, +) +from sds_gateway.api_methods.federation.compile_federated_data import ( + capture_in_published_dataset, +) +from sds_gateway.api_methods.federation.compile_federated_data import ( + compile_federated_doc, +) +from sds_gateway.api_methods.federation.compile_federated_data import ( + fed_doc_exists, ) -from sds_gateway.api_methods.helpers.compile_federated_data import ( +from sds_gateway.api_methods.federation.compile_federated_data import ( + fed_doc_is_tombstoned, +) +from sds_gateway.api_methods.federation.compile_federated_data import ( + federation_site_name, +) +from sds_gateway.api_methods.federation.compile_federated_data import ( is_federation_exportable_dataset, ) +from sds_gateway.api_methods.federation.events import FederationEventType +from sds_gateway.api_methods.federation.events import publish_federation_event +from sds_gateway.api_methods.federation.fed_index import LocalFederatedIndexer from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset from sds_gateway.api_methods.models import ItemType +from sds_gateway.api_methods.utils.opensearch_client import get_opensearch_client -def _event_type(*, created: bool, exportable: bool) -> str: - if not exportable: - return "deleted" - return "created" if created else "updated" +def _federation_indexing_enabled() -> bool: + if not getattr(settings, "FEDERATION_ENABLED", False): + return False + return bool(federation_site_name()) -def _skip_signal() -> bool: - if not getattr(settings, "FEDERATION_ENABLED", False): - log.debug("FEDERATION_ENABLED is False, skipping federation signal") +def _event_type(*, created: bool, deleted: bool) -> FederationEventType: + if deleted: + return FederationEventType.DELETED + if created: + return FederationEventType.CREATED + return FederationEventType.UPDATED + + +def _federation_created_flag( + instance: Dataset | Capture, + item_type: ItemType, +) -> bool: + site_name = federation_site_name() + if not fed_doc_exists( + asset_type=item_type, + site_name=site_name, + instance=instance, + ): return True + return fed_doc_is_tombstoned( + asset_type=item_type, + site_name=site_name, + instance=instance, + ) - if not is_federation_operational(): - log.debug("Federation not operational, skipping signal") + +def _ignore_stale_deleted_update( + instance: Dataset | Capture, + item_type: ItemType, +) -> bool: + if not instance.is_deleted: + return False + site_name = federation_site_name() + if not fed_doc_exists( + asset_type=item_type, + site_name=site_name, + instance=instance, + ): return True + return fed_doc_is_tombstoned( + asset_type=item_type, + site_name=site_name, + instance=instance, + ) - return False + +def _index_and_notify( + *, + instance: Dataset | Capture, + item_type: ItemType, + created: bool, + deleted: bool, +) -> None: + if not _federation_indexing_enabled(): + log.debug("Federation indexing disabled; skipping signal") + return + + site_name = federation_site_name() + event_type = _event_type(created=created, deleted=deleted) + timestamp = instance.updated_at + body = compile_federated_doc(instance) + + try: + LocalFederatedIndexer(get_opensearch_client()).apply_local_event( + event_type=str(event_type), + event_at=timestamp, + site_name=site_name, + item_type=item_type, + uuid=instance.uuid, + body=body, + ) + except Exception as exc: # noqa: BLE001 + log.warning("Federation OpenSearch index failed: {}", exc) + return + + publish_federation_event( + event_type=event_type, + item_type=item_type, + uuid=instance.uuid, + timestamp=timestamp, + ) + + +def _tombstone_federated_if_exists( + instance: Dataset | Capture, + item_type: ItemType, +) -> None: + site_name = federation_site_name() + if not fed_doc_exists( + asset_type=item_type, + site_name=site_name, + instance=instance, + ): + return + if fed_doc_is_tombstoned( + asset_type=item_type, + site_name=site_name, + instance=instance, + ): + return + _index_and_notify( + instance=instance, + item_type=item_type, + created=False, + deleted=True, + ) + + +def _upsert_federated(instance: Dataset | Capture, item_type: ItemType) -> None: + created = _federation_created_flag(instance, item_type) + _index_and_notify( + instance=instance, + item_type=item_type, + created=created, + deleted=False, + ) + + +def _maybe_tombstone_capture_after_dataset_change( + capture: Capture, + *, + dataset: Dataset, +) -> None: + if _ignore_stale_deleted_update(capture, ItemType.CAPTURE): + return + if capture_in_other_published_datasets( + capture, + exclude_dataset_id=dataset.uuid, + ): + return + _tombstone_federated_if_exists(capture, ItemType.CAPTURE) + + +def _sync_published_captures_for_dataset(dataset: Dataset) -> None: + for capture in dataset.captures.filter(is_deleted=False): + if not capture_in_published_dataset(capture): + continue + if _ignore_stale_deleted_update(capture, ItemType.CAPTURE): + continue + _upsert_federated(capture, ItemType.CAPTURE) @receiver(post_save, sender=Dataset) @@ -45,32 +192,40 @@ def federation_dataset_changed( created: bool, # noqa: FBT001 **kwargs, ) -> None: - if _skip_signal(): + if not _federation_indexing_enabled(): + return + if _ignore_stale_deleted_update(instance, ItemType.DATASET): return - exportable = is_federation_exportable_dataset(instance) - publish_federation_event( - event_type=_event_type(created=created, exportable=exportable), - item_type=ItemType.DATASET, - uuid=instance.uuid, - timestamp=instance.updated_at, - ) + if instance.is_deleted: + _tombstone_federated_if_exists(instance, ItemType.DATASET) + for capture in instance.captures.all(): + _maybe_tombstone_capture_after_dataset_change(capture, dataset=instance) + return + + if not is_federation_exportable_dataset(instance): + return + + _upsert_federated(instance, ItemType.DATASET) + _sync_published_captures_for_dataset(instance) @receiver(post_save, sender=Capture) def federation_capture_changed( sender: type[Capture], instance: Capture, - created: bool, # noqa: FBT001 **kwargs, ) -> None: - if _skip_signal(): + if not _federation_indexing_enabled(): + return + if _ignore_stale_deleted_update(instance, ItemType.CAPTURE): return - exportable = is_federation_exportable_capture(instance) - publish_federation_event( - event_type=_event_type(created=created, exportable=exportable), - item_type=ItemType.CAPTURE, - uuid=instance.uuid, - timestamp=instance.updated_at, - ) + if instance.is_deleted: + _tombstone_federated_if_exists(instance, ItemType.CAPTURE) + return + + if not capture_in_published_dataset(instance): + return + + _upsert_federated(instance, ItemType.CAPTURE) diff --git a/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py b/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py deleted file mode 100644 index 53be29f53..000000000 --- a/gateway/sds_gateway/api_methods/helpers/compile_federated_data.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Build federation export payloads (RFC fed-* index shape).""" - -from __future__ import annotations - -from typing import TYPE_CHECKING -from typing import Any - -from django.conf import settings - -from sds_gateway.api_methods.models import Capture -from sds_gateway.api_methods.models import Dataset -from sds_gateway.api_methods.models import DatasetStatus -from sds_gateway.api_methods.serializers.capture_serializers import ( - CaptureFederationSerializer, -) -from sds_gateway.api_methods.serializers.dataset_serializers import ( - DatasetFederationSerializer, -) - -if TYPE_CHECKING: - from django.db.models import QuerySet - - -def federation_site_name() -> str: - return getattr(settings, "FEDERATION_SITE_NAME", "").strip() - - -def public_datasets_queryset() -> QuerySet[Dataset]: - return ( - Dataset.objects.filter( - status=DatasetStatus.FINAL, - is_public=True, - is_deleted=False, - ) - .prefetch_related("keywords", "owner") - .order_by("-updated_at") - ) - - -def public_captures_queryset() -> QuerySet[Capture]: - return Capture.objects.filter(is_public=True, is_deleted=False).order_by( - "-updated_at", - ) - - -def is_federation_exportable_dataset(dataset: Dataset) -> bool: - return ( - not dataset.is_deleted - and dataset.is_public - and dataset.status == DatasetStatus.FINAL - ) - - -def is_federation_exportable_capture(capture: Capture) -> bool: - return not capture.is_deleted and capture.is_public - - -def compile_federated_dataset_doc(dataset: Dataset) -> dict[str, Any]: - """Serialize a public dataset for federation sync / OpenSearch.""" - site_name = federation_site_name() - federation_data = DatasetFederationSerializer( - dataset, - context={"site_name": site_name}, - ) - return federation_data.data - - -def compile_federated_capture_doc(capture: Capture) -> dict[str, Any]: - """Serialize a public capture for federation sync / OpenSearch.""" - site_name = federation_site_name() - federation_data = CaptureFederationSerializer( - capture, - context={"site_name": site_name}, - ) - return federation_data.data diff --git a/gateway/sds_gateway/api_methods/models.py b/gateway/sds_gateway/api_methods/models.py index 322fef020..eb290457e 100644 --- a/gateway/sds_gateway/api_methods/models.py +++ b/gateway/sds_gateway/api_methods/models.py @@ -12,6 +12,7 @@ from blake3 import blake3 as Blake3 # noqa: N812 from django.conf import settings +from django.core.exceptions import ValidationError from django.core.signals import request_started from django.db import models from django.db.models import Count @@ -98,6 +99,10 @@ class ItemType(StrEnum): CAPTURE = "capture" FILE = "file" + def pluralize(self) -> str: + """Get the plural form of the item type.""" + return self.value + "s" + class ProcessingType(StrEnum): """The type of post-processing.""" @@ -985,6 +990,42 @@ class Dataset(BaseModel): def __str__(self) -> str: return self.name + def _published_snapshot_from_db(self) -> tuple[str, bool] | None: + if not self.pk: + return None + row = ( + type(self).objects.filter(pk=self.pk) + .values_list("status", "is_public") + .first() + ) + return row # (status, is_public) or None + + @staticmethod + def _was_final_status(status: DatasetStatus) -> bool: + return status == DatasetStatus.FINAL + + @staticmethod + def _was_public(is_public: bool) -> bool: + return is_public + + def clean(self) -> None: + super().clean() + snapshot = self._published_snapshot_from_db() + if snapshot is None: + return + + old_status, old_is_public = snapshot + if not self._was_final_status(old_status) and not self._was_public( + is_public=old_is_public, + ): + return + + if self._was_final_status(status=old_status) and self.status != old_status: + raise ValidationError("Final datasets cannot be reverted to draft.") + + if self._was_public(is_public=old_is_public) and self.is_public != old_is_public: + raise ValidationError("Public datasets cannot be reverted to private.") + def save(self, *args, **kwargs) -> None: # Serialize the list fields to a JSON string before saving for field in self.list_fields: @@ -1019,6 +1060,7 @@ def save(self, *args, **kwargs) -> None: ) raise ValueError(msg) + self.full_clean() super().save(*args, **kwargs) def soft_delete(self) -> None: diff --git a/gateway/sds_gateway/api_methods/tests/test_dataset_publish_immutability.py b/gateway/sds_gateway/api_methods/tests/test_dataset_publish_immutability.py new file mode 100644 index 000000000..4283851b6 --- /dev/null +++ b/gateway/sds_gateway/api_methods/tests/test_dataset_publish_immutability.py @@ -0,0 +1,45 @@ +"""Tests for Dataset publish immutability (status final / is_public).""" + +from __future__ import annotations + +import pytest +from django.core.exceptions import ValidationError + +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.tests.factories import DatasetFactory + +pytestmark = pytest.mark.django_db + + +class TestDatasetPublishImmutability: + def test_cannot_revert_final_to_draft(self) -> None: + dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=False) + dataset.status = DatasetStatus.DRAFT + with pytest.raises(ValidationError, match="cannot be reverted to draft"): + dataset.save() + + def test_cannot_make_public_dataset_private(self) -> None: + dataset = DatasetFactory(status=DatasetStatus.DRAFT, is_public=True) + dataset.is_public = False + with pytest.raises(ValidationError, match="cannot be reverted to private"): + dataset.save() + + def test_final_and_public_can_update_other_fields(self) -> None: + dataset = DatasetFactory( + status=DatasetStatus.FINAL, + is_public=True, + name="Original", + ) + dataset.name = "Updated title" + dataset.save() + dataset.refresh_from_db() + assert dataset.name == "Updated title" + assert dataset.status == DatasetStatus.FINAL + assert dataset.is_public is True + + def test_draft_can_still_become_final(self) -> None: + dataset = DatasetFactory(status=DatasetStatus.DRAFT, is_public=False) + dataset.status = DatasetStatus.FINAL + dataset.save() + dataset.refresh_from_db() + assert dataset.status == DatasetStatus.FINAL diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_events.py b/gateway/sds_gateway/api_methods/tests/test_federation_events.py index 62f458105..b0e0fd068 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_events.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_events.py @@ -80,3 +80,46 @@ def test_skips_publish_when_not_operational( uuid=uuid4(), ) mock_get_redis.assert_not_called() + + +class TestFederationSignals: + @override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", + FEDERATION_OPERATIONAL_OVERRIDE=True, + FEDERATION_EVENTS_CHANNEL="federation:events:crc", + ) + @patch("sds_gateway.api_methods.federation.signals.publish_federation_event") + @patch("sds_gateway.api_methods.federation.signals.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.signals.get_opensearch_client") + @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=False) + def test_dataset_post_save_indexes_when_published( + self, + _mock_exists: MagicMock, + _mock_os: MagicMock, + mock_indexer_cls: MagicMock, + mock_publish: MagicMock, + ) -> None: + from sds_gateway.api_methods.models import Dataset + from sds_gateway.api_methods.models import DatasetStatus + from sds_gateway.api_methods.federation.signals import federation_dataset_changed + from sds_gateway.api_methods.tests.factories import DatasetFactory + + mock_indexer = mock_indexer_cls.return_value + dataset = DatasetFactory( + status=DatasetStatus.FINAL, + is_public=True, + ) + federation_dataset_changed( + sender=Dataset, + instance=dataset, + created=False, + ) + + mock_indexer.apply_local_event.assert_called_once() + call = mock_indexer.apply_local_event.call_args.kwargs + assert call["site_name"] == "crc" + assert call["item_type"] == ItemType.DATASET + assert call["uuid"] == dataset.uuid + assert call["event_type"] == "created" + mock_publish.assert_called_once() diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export.py b/gateway/sds_gateway/api_methods/tests/test_federation_export.py index 9dfe2a864..9e2503f27 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export.py @@ -1,5 +1,7 @@ """Tests for federation export endpoints and API key scoping.""" +from unittest.mock import patch + from django.contrib.auth import get_user_model from django.test import override_settings from django.urls import reverse @@ -8,6 +10,9 @@ from sds_gateway.api_methods.models import DatasetStatus from sds_gateway.api_methods.models import KeySources +from sds_gateway.api_methods.federation.compile_federated_data import ( + compile_federated_dataset_doc, +) from sds_gateway.api_methods.tests.factories import CaptureFactory from sds_gateway.api_methods.tests.factories import DatasetFactory from sds_gateway.users.models import UserAPIKey @@ -77,15 +82,34 @@ def test_sync_key_can_list_public_datasets(self) -> None: assert str(self.private_dataset.uuid) not in uuids def test_sync_key_can_retrieve_public_dataset(self) -> None: - response = self.client.get( - self.detail_dataset_url, - REMOTE_ADDR="127.0.0.1", - **self._auth(self.sync_key), - ) + indexed = compile_federated_dataset_doc(self.public_dataset) + with patch( + "sds_gateway.api_methods.views.federation_endpoints." + "get_federated_export_doc_by_uuid", + return_value=indexed, + ): + response = self.client.get( + self.detail_dataset_url, + REMOTE_ADDR="127.0.0.1", + **self._auth(self.sync_key), + ) assert response.status_code == status.HTTP_200_OK assert response.json()["uuid"] == str(self.public_dataset.uuid) assert response.json()["site_name"] == "crc" + def test_sync_key_dataset_detail_404_when_not_indexed(self) -> None: + with patch( + "sds_gateway.api_methods.views.federation_endpoints." + "get_federated_export_doc_by_uuid", + return_value=None, + ): + response = self.client.get( + self.detail_dataset_url, + REMOTE_ADDR="127.0.0.1", + **self._auth(self.sync_key), + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + def test_regular_key_denied_on_export(self) -> None: response = self.client.get( self.list_datasets_url, diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py index 910e0731c..3e4a93e2a 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py @@ -9,10 +9,10 @@ from django.contrib.auth import get_user_model from sds_gateway.api_methods.federation.export_contract import assert_field_names_match -from sds_gateway.api_methods.helpers.compile_federated_data import ( +from sds_gateway.api_methods.federation.compile_federated_data import ( compile_federated_capture_doc, ) -from sds_gateway.api_methods.helpers.compile_federated_data import ( +from sds_gateway.api_methods.federation.compile_federated_data import ( compile_federated_dataset_doc, ) from sds_gateway.api_methods.models import DatasetStatus diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py new file mode 100644 index 000000000..f34c20484 --- /dev/null +++ b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py @@ -0,0 +1,248 @@ +"""Tests for federation signal orchestration.""" + +from __future__ import annotations + +from unittest.mock import MagicMock +from unittest.mock import patch + +import pytest +from django.test import TestCase +from django.test import override_settings + +from sds_gateway.api_methods.federation.signals import federation_capture_changed +from sds_gateway.api_methods.federation.signals import federation_dataset_changed +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.models import ItemType +from sds_gateway.api_methods.tests.factories import CaptureFactory +from sds_gateway.api_methods.tests.factories import DatasetFactory + +pytestmark = pytest.mark.django_db + + +@override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", + FEDERATION_OPERATIONAL_OVERRIDE=True, +) +class TestFederationDatasetSignals(TestCase): + @patch("sds_gateway.api_methods.federation.signals.publish_federation_event") + @patch("sds_gateway.api_methods.federation.signals.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.signals.get_opensearch_client") + @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=False) + def test_published_dataset_upserts_and_syncs_captures( + self, + _mock_exists: MagicMock, + _mock_os: MagicMock, + mock_indexer_cls: MagicMock, + mock_publish: MagicMock, + ) -> None: + mock_indexer = mock_indexer_cls.return_value + dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) + capture = CaptureFactory(is_deleted=False) + capture.datasets.add(dataset) + + federation_dataset_changed( + sender=Dataset, + instance=dataset, + created=False, + ) + + assert mock_indexer.apply_local_event.call_count == 2 + item_types = { + c.kwargs["item_type"] + for c in mock_indexer.apply_local_event.call_args_list + } + assert item_types == {ItemType.DATASET, ItemType.CAPTURE} + assert mock_publish.call_count == 2 + + @patch("sds_gateway.api_methods.federation.signals.publish_federation_event") + @patch("sds_gateway.api_methods.federation.signals.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.signals.get_opensearch_client") + @patch( + "sds_gateway.api_methods.federation.signals.fed_doc_exists", + return_value=True, + ) + @patch( + "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", + return_value=False, + ) + def test_deleted_dataset_tombstones_when_doc_exists( + self, + _mock_tombstone: MagicMock, + _mock_exists: MagicMock, + _mock_os: MagicMock, + mock_indexer_cls: MagicMock, + mock_publish: MagicMock, + ) -> None: + mock_indexer = mock_indexer_cls.return_value + dataset = DatasetFactory( + status=DatasetStatus.FINAL, + is_public=True, + is_deleted=True, + ) + + federation_dataset_changed( + sender=Dataset, + instance=dataset, + created=False, + ) + + mock_indexer.apply_local_event.assert_called() + deleted_calls = [ + c + for c in mock_indexer.apply_local_event.call_args_list + if c.kwargs.get("event_type") == "deleted" + ] + assert deleted_calls + + @patch("sds_gateway.api_methods.federation.signals._upsert_federated") + @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=False) + def test_draft_dataset_skips_federation( + self, + _mock_exists: MagicMock, + mock_upsert: MagicMock, + ) -> None: + dataset = DatasetFactory(status=DatasetStatus.DRAFT, is_public=False) + federation_dataset_changed(sender=Dataset, instance=dataset, created=True) + mock_upsert.assert_not_called() + + @patch("sds_gateway.api_methods.federation.signals._tombstone_federated_if_exists") + @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=True) + @patch( + "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", + return_value=False, + ) + def test_deleted_dataset_skips_capture_tombstone_when_on_other_published( + self, + _mock_tombstone_flag: MagicMock, + _mock_exists: MagicMock, + mock_tombstone: MagicMock, + ) -> None: + dataset_a = DatasetFactory( + status=DatasetStatus.FINAL, + is_public=True, + is_deleted=True, + ) + dataset_b = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) + capture = CaptureFactory(is_deleted=False) + capture.datasets.add(dataset_a, dataset_b) + + federation_dataset_changed( + sender=Dataset, + instance=dataset_a, + created=False, + ) + + tombstone_calls = [ + c for c in mock_tombstone.call_args_list if c.args[1] == ItemType.DATASET + ] + assert len(tombstone_calls) == 1 + capture_tombstones = [ + c for c in mock_tombstone.call_args_list if c.args[1] == ItemType.CAPTURE + ] + assert not capture_tombstones + + @patch("sds_gateway.api_methods.federation.signals._tombstone_federated_if_exists") + @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=True) + @patch( + "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", + return_value=False, + ) + def test_deleted_dataset_tombstones_orphan_capture( + self, + _mock_tombstone_flag: MagicMock, + _mock_exists: MagicMock, + mock_tombstone: MagicMock, + ) -> None: + dataset = DatasetFactory( + status=DatasetStatus.FINAL, + is_public=True, + is_deleted=True, + ) + capture = CaptureFactory(is_deleted=False) + capture.datasets.add(dataset) + + federation_dataset_changed( + sender=Dataset, + instance=dataset, + created=False, + ) + + assert mock_tombstone.call_count == 2 + + +@override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", + FEDERATION_OPERATIONAL_OVERRIDE=True, +) +class TestFederationCaptureSignals(TestCase): + @patch("sds_gateway.api_methods.federation.signals.publish_federation_event") + @patch("sds_gateway.api_methods.federation.signals.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.signals.get_opensearch_client") + @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=False) + def test_capture_on_published_dataset_upserts( + self, + _mock_exists: MagicMock, + _mock_os: MagicMock, + mock_indexer_cls: MagicMock, + mock_publish: MagicMock, + ) -> None: + mock_indexer = mock_indexer_cls.return_value + dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) + capture = CaptureFactory(is_deleted=False) + capture.datasets.add(dataset) + + federation_capture_changed(sender=Capture, instance=capture) + + mock_indexer.apply_local_event.assert_called_once() + assert ( + mock_indexer.apply_local_event.call_args.kwargs["item_type"] + == ItemType.CAPTURE + ) + mock_publish.assert_called_once() + + @patch("sds_gateway.api_methods.federation.signals._upsert_federated") + def test_capture_on_draft_dataset_skips_upsert( + self, + mock_upsert: MagicMock, + ) -> None: + dataset = DatasetFactory(status=DatasetStatus.DRAFT, is_public=False) + capture = CaptureFactory(is_deleted=False) + capture.datasets.add(dataset) + federation_capture_changed(sender=Capture, instance=capture) + mock_upsert.assert_not_called() + + @patch("sds_gateway.api_methods.federation.signals._tombstone_federated_if_exists") + @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=True) + @patch( + "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", + return_value=False, + ) + def test_deleted_capture_tombstones_when_doc_exists( + self, + _mock_tombstone_flag: MagicMock, + _mock_exists: MagicMock, + mock_tombstone: MagicMock, + ) -> None: + capture = CaptureFactory(is_deleted=True) + federation_capture_changed(sender=Capture, instance=capture) + mock_tombstone.assert_called_once_with(capture, ItemType.CAPTURE) + + @patch("sds_gateway.api_methods.federation.signals._tombstone_federated_if_exists") + @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=True) + @patch( + "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", + return_value=True, + ) + def test_stale_deleted_capture_ignored( + self, + _mock_tombstone_flag: MagicMock, + _mock_exists: MagicMock, + mock_tombstone: MagicMock, + ) -> None: + capture = CaptureFactory(is_deleted=True) + federation_capture_changed(sender=Capture, instance=capture) + mock_tombstone.assert_not_called() diff --git a/gateway/sds_gateway/api_methods/views/federation_endpoints.py b/gateway/sds_gateway/api_methods/views/federation_endpoints.py index d9a215a26..d02192a13 100644 --- a/gateway/sds_gateway/api_methods/views/federation_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/federation_endpoints.py @@ -1,12 +1,13 @@ -"""Federation export API (sync service → gateway, Postgres public metadata).""" +"""Federation export API (sync service reads local fed-* OpenSearch docs).""" from __future__ import annotations from typing import TYPE_CHECKING +from uuid import UUID -from django.shortcuts import get_object_or_404 from drf_spectacular.utils import extend_schema from rest_framework.decorators import action +from rest_framework.exceptions import NotFound from rest_framework.response import Response from rest_framework.viewsets import ViewSet @@ -14,33 +15,38 @@ from rest_framework.request import Request from sds_gateway.api_methods.authentication import APIKeyAuthentication -from sds_gateway.api_methods.federation.permissions import ( - IsFederationInternalExportClient, -) -from sds_gateway.api_methods.federation.permissions import IsFederationOperational -from sds_gateway.api_methods.helpers.compile_federated_data import ( +from sds_gateway.api_methods.federation.compile_federated_data import ( compile_federated_capture_doc, ) -from sds_gateway.api_methods.helpers.compile_federated_data import ( +from sds_gateway.api_methods.federation.compile_federated_data import ( compile_federated_dataset_doc, ) -from sds_gateway.api_methods.helpers.compile_federated_data import ( - is_federation_exportable_capture, +from sds_gateway.api_methods.federation.compile_federated_data import ( + get_federated_export_doc_by_uuid, ) -from sds_gateway.api_methods.helpers.compile_federated_data import ( - is_federation_exportable_dataset, -) -from sds_gateway.api_methods.helpers.compile_federated_data import ( +from sds_gateway.api_methods.federation.compile_federated_data import ( public_captures_queryset, ) -from sds_gateway.api_methods.helpers.compile_federated_data import ( +from sds_gateway.api_methods.federation.compile_federated_data import ( public_datasets_queryset, ) -from sds_gateway.api_methods.models import Capture -from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.federation.permissions import ( + IsFederationInternalExportClient, +) +from sds_gateway.api_methods.federation.permissions import IsFederationOperational +from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.permissions import IsFederationSyncKey +def _federation_export_uuid(pk: str | None) -> UUID: + if not pk: + raise NotFound() + try: + return UUID(pk) + except ValueError as exc: + raise NotFound() from exc + + @extend_schema(exclude=True) class FederationViewSet(ViewSet): """Internal export endpoints for the federation sync service.""" @@ -68,14 +74,15 @@ def export_datasets_list(self, request: Request) -> Response: def export_dataset_detail( self, request: Request, pk: str | None = None ) -> Response: - """Return one public dataset for sync after a local Redis event.""" - dataset = get_object_or_404(Dataset, pk=pk, is_deleted=False) - if not is_federation_exportable_dataset(dataset): - return Response( - {"detail": "Dataset is not available for federation export."}, - status=404, - ) - return Response(compile_federated_dataset_doc(dataset)) + """Return one indexed dataset doc (post-signal); 404 if not in fed-datasets.""" + asset_uuid = _federation_export_uuid(pk) + body = get_federated_export_doc_by_uuid( + asset_type=ItemType.DATASET, + asset_uuid=asset_uuid, + ) + if body is None: + raise NotFound() + return Response(body) @action(detail=False, methods=["get"], url_path="export/captures") def export_captures_list(self, request: Request) -> Response: @@ -93,11 +100,12 @@ def export_captures_list(self, request: Request) -> Response: def export_capture_detail( self, request: Request, pk: str | None = None ) -> Response: - """Return one public capture for sync after a local Redis event.""" - capture = get_object_or_404(Capture, pk=pk, is_deleted=False) - if not is_federation_exportable_capture(capture): - return Response( - {"detail": "Capture is not available for federation export."}, - status=404, - ) - return Response(compile_federated_capture_doc(capture)) + """Return one indexed capture doc (post-signal); 404 if not in fed-captures.""" + asset_uuid = _federation_export_uuid(pk) + body = get_federated_export_doc_by_uuid( + asset_type=ItemType.CAPTURE, + asset_uuid=asset_uuid, + ) + if body is None: + raise NotFound() + return Response(body) From c39a16c124ceb92a6e50ef7554f534e386323067 Mon Sep 17 00:00:00 2001 From: klpoland Date: Thu, 9 Jul 2026 15:20:25 -0400 Subject: [PATCH 10/17] fix: cidrs list, sync service key generation allign with svi methods (TODO: implement on federation app side), federation init/checks refactor, federation queryset reusability --- gateway/.envs/example/django.env | 27 ++-- gateway/config/settings/base.py | 55 +++++-- gateway/sds_gateway/api_methods/apps.py | 27 +++- .../api_methods/federation/availability.py | 39 ++++- .../federation/compile_federated_data.py | 34 +---- .../api_methods/federation/signals.py | 5 +- .../management/commands/prepare_gateway.py | 15 +- gateway/sds_gateway/api_methods/models.py | 21 +++ .../tests/test_federation_export.py | 5 +- .../tests/test_federation_hardening.py | 52 +++++++ .../tests/test_federation_sync_api_key.py | 74 +++++++++ gateway/sds_gateway/users/api/views.py | 51 ++++--- .../users/backend_service_key_utils.py | 101 +++++++++++++ .../create_federation_sync_api_key.py | 46 ------ .../management/commands/create_test_files.py | 140 ------------------ .../commands/init_federation_sync_token.py | 23 +++ .../management/commands/init_svi_token.py | 2 +- gateway/sds_gateway/users/svi_utils.py | 53 ------- .../sds_gateway/users/tests/test_svi_utils.py | 2 +- gateway/sds_gateway/users/urls.py | 10 +- 20 files changed, 456 insertions(+), 326 deletions(-) create mode 100644 gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py create mode 100644 gateway/sds_gateway/users/backend_service_key_utils.py delete mode 100644 gateway/sds_gateway/users/management/commands/create_federation_sync_api_key.py delete mode 100644 gateway/sds_gateway/users/management/commands/create_test_files.py create mode 100644 gateway/sds_gateway/users/management/commands/init_federation_sync_token.py delete mode 100644 gateway/sds_gateway/users/svi_utils.py diff --git a/gateway/.envs/example/django.env b/gateway/.envs/example/django.env index f306a3f6b..f4f4607a6 100644 --- a/gateway/.envs/example/django.env +++ b/gateway/.envs/example/django.env @@ -10,16 +10,23 @@ API_KEY= # FEDERATION # ------------------------------------------------------------------------------ -# Peer sync uses the federation-sync Docker service (same sds-network as gateway; -# service definition lives under /federation). Bootstrap: enable federation, run -# create_federation_sync_api_key, pass the key to federation-sync. Set FEDERATION_SITE_NAME -# (e.g. crc) when enabling federation; use SDS_SITE_FQDN for the public host (RFC [site].fqdn). -# FEDERATION_ENABLED=true # Master switch for export APIs and Redis federation events. -# FEDERATION_SITE_NAME=crc # RFC [site].name (short peer id); set SDS_SITE_FQDN separately for [site].fqdn. -# FEDERATION_EVENTS_CHANNEL=federation:events # Redis pub/sub channel federation-sync subscribes to. -# FEDERATION_SYNC_HEALTH_URL=http://federation-sync:8000/sync/health # Health probe target (federation-sync service). -# FEDERATION_SYNC_USER_EMAIL=federation-sync@internal.local # Service user email for create_federation_sync_api_key. -# FEDERATION_EXPORT_ALLOWED_CIDRS= # Comma-separated CIDRs allowed to call export (default: private Docker ranges). +# Peer sync: shared FEDERATION_SYNC_SERVER_API_KEY in django.env + federation sync.env; +# prepare_gateway runs init_federation_sync_token. Sync startup calls +# GET /users/get-federation-sync-api-key/ (Token auth) and writes FEDERATION_GATEWAY_API_KEY locally. +# Set FEDERATION_SITE_NAME (e.g. crc) when enabling federation; use SDS_SITE_FQDN for the public host (RFC [site].fqdn). +# Master switch for export APIs and Redis federation events. +# FEDERATION_ENABLED=true +# RFC [site].name (short peer id); set SDS_SITE_FQDN separately for [site].fqdn. +# FEDERATION_SITE_NAME=crc +# Redis pub/sub channel federation-sync subscribes to. +# FEDERATION_EVENTS_CHANNEL=federation:events +# Health probe target (federation-sync service). +# FEDERATION_SYNC_HEALTH_URL=http://federation-sync:8000/sync/health +# FEDERATION_SYNC_USER_EMAIL=federation-sync@internal.local +# Bootstrap Token (shared with federation sync.env); not the export Api-Key. +# FEDERATION_SYNC_SERVER_API_KEY= +# Comma-separated CIDRs allowed to call export (default: private Docker ranges). +# FEDERATION_EXPORT_ALLOWED_CIDRS= # AUTH0 # ------------------------------------------------------------------------------ diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index d176814f0..516c49ed9 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -27,8 +27,41 @@ def __get_random_token(length: int) -> str: __rng.choice(string.ascii_letters + string.digits) for _ in range(length) ) -def _parse_cidrs(raw: list[str]) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]: - return [ipaddress.ip_network(item.strip(), strict=False) for item in raw] + +FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT: list[str] = [ + "127.0.0.1/32", + "::1/128", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", +] + + +def _parse_cidrs( + raw: list[str], +) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]: + networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = [] + for item in raw: + text = str(item).strip() + if not text: + continue + networks.append(ipaddress.ip_network(text, strict=False)) + return networks + + +def _federation_export_allowed_cidrs_from_env() -> list[ + ipaddress.IPv4Network | ipaddress.IPv6Network +]: + raw = env.list( + "FEDERATION_EXPORT_ALLOWED_CIDRS", + default=FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT, + ) + tokens = [str(item).strip() for item in raw] + tokens = [token for token in tokens if token] + if not tokens: + tokens = list(FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT) + return _parse_cidrs(tokens) + env.read_env() @@ -734,6 +767,11 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: "FEDERATION_SYNC_USER_EMAIL", default="federation-sync@internal.local", ) +# Shared with federation-sync (Authorization: Token …) to call get-federation-sync-api-key. +FEDERATION_SYNC_SERVER_API_KEY: str = env.str( + "FEDERATION_SYNC_SERVER_API_KEY", + default=__get_random_token(TOKEN_LENGTH), +) FEDERATION_SYNC_HEALTH_URL: str = env.str( "FEDERATION_SYNC_HEALTH_URL", default="http://federation-sync:8000/sync/health", @@ -762,18 +800,7 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: # Export API: internal Docker/private networks (sync → django on sds-network). FEDERATION_EXPORT_ALLOWED_CIDRS: list[ ipaddress.IPv4Network | ipaddress.IPv6Network -] = _parse_cidrs( - env.list( - "FEDERATION_EXPORT_ALLOWED_CIDRS", - default=[ - "127.0.0.1/32", - "::1/128", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - ], - ), -) +] = _federation_export_allowed_cidrs_from_env() # ADMIN_CONSOLE_ENV is used to visually distinguish between different environments # (production, staging, local) in the admin console and error emails. It does not affect diff --git a/gateway/sds_gateway/api_methods/apps.py b/gateway/sds_gateway/api_methods/apps.py index 3d189bbef..023e95e5c 100644 --- a/gateway/sds_gateway/api_methods/apps.py +++ b/gateway/sds_gateway/api_methods/apps.py @@ -1,9 +1,24 @@ import logging +import sys from django.apps import AppConfig +from django.db.models.signals import post_migrate from loguru import logger as log +def _skip_federation_init_in_ready() -> bool: + """Management commands that load apps before the schema exists.""" + if len(sys.argv) < 2: # noqa: PLR2004 + return False + return sys.argv[1] in { + "migrate", + "makemigrations", + "showmigrations", + "sqlmigrate", + "flush", + } + + class ApiMethodsConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "sds_gateway.api_methods" @@ -18,7 +33,17 @@ def ready(self) -> None: initialize_federation_operational_state, ) - initialize_federation_operational_state() + def _init_federation_after_migrate(_sender, **kwargs) -> None: + initialize_federation_operational_state() + + post_migrate.connect( + _init_federation_after_migrate, + sender=self, + dispatch_uid="api_methods_federation_operational_init", + ) + + if not _skip_federation_init_in_ready(): + initialize_federation_operational_state() silence_unwanted_logs() diff --git a/gateway/sds_gateway/api_methods/federation/availability.py b/gateway/sds_gateway/api_methods/federation/availability.py index 75cd6fd39..eaaf32e59 100644 --- a/gateway/sds_gateway/api_methods/federation/availability.py +++ b/gateway/sds_gateway/api_methods/federation/availability.py @@ -10,6 +10,7 @@ from typing import Any from django.conf import settings +from django.db.utils import DatabaseError from loguru import logger as log from sds_gateway.api_methods.models import KeySources @@ -34,8 +35,16 @@ def _export_allowed_networks() -> list[ipaddress.IPv4Network | ipaddress.IPv6Net for item in raw: if isinstance(item, (ipaddress.IPv4Network, ipaddress.IPv6Network)): networks.append(item) - else: - networks.append(ipaddress.ip_network(str(item).strip(), strict=False)) + continue + text = str(item).strip() + if not text: + continue + networks.append(ipaddress.ip_network(text, strict=False)) + if not networks: + from config.settings.base import FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT + from config.settings.base import _parse_cidrs + + networks = _parse_cidrs(FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT) return networks @@ -102,7 +111,10 @@ def _sync_health_ok() -> tuple[bool, str]: # noqa: PLR0911 def _sync_api_key_present() -> tuple[bool, str]: if _setting("FEDERATION_SKIP_SYNC_API_KEY_CHECK", default=False): return True, "sync API key check skipped" - exists = UserAPIKey.objects.filter(source=KeySources.FederationSync).exists() + exists = UserAPIKey.objects.filter( + source=KeySources.FederationSync, + revoked=False, + ).exists() if not exists: return False, "no FederationSync API key in database" return True, "FederationSync API key present" @@ -130,7 +142,7 @@ def evaluate_federation_operational() -> tuple[bool, str]: if not site_name: return False, "FEDERATION_SITE_NAME must be set when federation is enabled" - for check in (_sync_api_key_present, _sync_health_ok, _redis_ok): + for check in (_sync_api_key_present, _redis_ok, _sync_health_ok): ok, reason = check() if not ok: return False, reason @@ -157,7 +169,26 @@ def refresh_federation_operational_state(*, force: bool = False) -> tuple[bool, return operational, reason +def federation_operational_db_ready() -> bool: + """False when federation probes would hit tables that are not migrated yet.""" + if not _setting("FEDERATION_ENABLED", default=False): + return True + table = UserAPIKey._meta.db_table + try: + from django.db import connection + + return table in connection.introspection.table_names() + except DatabaseError: + return False + + def initialize_federation_operational_state() -> None: + if not federation_operational_db_ready(): + settings.FEDERATION_OPERATIONAL = False + settings.FEDERATION_OPERATIONAL_REASON = "database tables not ready" + log.debug("Federation operational init deferred until migrations apply") + return + operational, reason = refresh_federation_operational_state(force=True) if operational: log.info("Federation is operational: {}", reason) diff --git a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py index 7e8fe166a..ca68689cf 100644 --- a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py +++ b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py @@ -13,7 +13,6 @@ from sds_gateway.api_methods.federation.fed_index import federated_doc_id from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset -from sds_gateway.api_methods.models import DatasetStatus from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.serializers.capture_serializers import ( CaptureFederationSerializer, @@ -31,20 +30,8 @@ def federation_site_name() -> str: return getattr(settings, "FEDERATION_SITE_NAME", "").strip() -def is_federation_exportable_dataset(dataset: Dataset) -> bool: - return ( - not dataset.is_deleted - and dataset.is_public - and dataset.status == DatasetStatus.FINAL - ) - - def capture_in_published_dataset(capture: Capture) -> bool: - return capture.datasets.filter( - status=DatasetStatus.FINAL, - is_public=True, - is_deleted=False, - ).exists() + return capture.datasets.federation_exportable().exists() def capture_in_other_published_datasets( @@ -53,11 +40,7 @@ def capture_in_other_published_datasets( exclude_dataset_id: UUID, ) -> bool: return ( - capture.datasets.filter( - status=DatasetStatus.FINAL, - is_public=True, - is_deleted=False, - ) + capture.datasets.federation_exportable() .exclude(uuid=exclude_dataset_id) .exists() ) @@ -144,17 +127,16 @@ def fed_doc_is_tombstoned( def public_datasets_queryset() -> QuerySet[Dataset]: return ( - Dataset.objects.filter( - status=DatasetStatus.FINAL, - is_public=True, - is_deleted=False, - ) + Dataset.objects.federation_exportable() .prefetch_related("keywords", "owner") .order_by("-updated_at") ) def public_captures_queryset() -> QuerySet[Capture]: - return Capture.objects.filter(is_public=True, is_deleted=False).order_by( - "-updated_at", + return ( + Capture.objects.filter(is_deleted=False) + .filter(datasets__in=Dataset.objects.federation_exportable()) + .distinct() + .order_by("-updated_at") ) diff --git a/gateway/sds_gateway/api_methods/federation/signals.py b/gateway/sds_gateway/api_methods/federation/signals.py index 9b47aa809..77b9471b8 100644 --- a/gateway/sds_gateway/api_methods/federation/signals.py +++ b/gateway/sds_gateway/api_methods/federation/signals.py @@ -25,9 +25,6 @@ from sds_gateway.api_methods.federation.compile_federated_data import ( federation_site_name, ) -from sds_gateway.api_methods.federation.compile_federated_data import ( - is_federation_exportable_dataset, -) from sds_gateway.api_methods.federation.events import FederationEventType from sds_gateway.api_methods.federation.events import publish_federation_event from sds_gateway.api_methods.federation.fed_index import LocalFederatedIndexer @@ -203,7 +200,7 @@ def federation_dataset_changed( _maybe_tombstone_capture_after_dataset_change(capture, dataset=instance) return - if not is_federation_exportable_dataset(instance): + if not instance.is_federation_exportable(): return _upsert_federated(instance, ItemType.DATASET) diff --git a/gateway/sds_gateway/api_methods/management/commands/prepare_gateway.py b/gateway/sds_gateway/api_methods/management/commands/prepare_gateway.py index 5f96e7189..c1c46bfe9 100644 --- a/gateway/sds_gateway/api_methods/management/commands/prepare_gateway.py +++ b/gateway/sds_gateway/api_methods/management/commands/prepare_gateway.py @@ -1,5 +1,6 @@ """Run gateway startup preparation in a single Django process.""" +from django.conf import settings from django.core.management import call_command from django.core.management.base import BaseCommand @@ -13,7 +14,7 @@ class Command(BaseCommand): help = ( "Prepare the gateway for serving: migrate, init search indices, " - "SVI token, and visualization pipelines." + "service tokens, and visualization pipelines." ) def add_arguments(self, parser): @@ -41,12 +42,22 @@ def handle(self, *args, **options) -> None: ("Applying database migrations...", "migrate", {"no_input": True}), ("Initializing OpenSearch indices...", "init_indices", {}), ("Initializing SVI server token...", "init_svi_token", {}), + ] + if getattr(settings, "FEDERATION_ENABLED", False): + steps.append( + ( + "Initializing federation sync server token...", + "init_federation_sync_token", + {}, + ), + ) + steps.append( ( "Setting up django-cog pipelines...", "setup_pipelines", {"strategy": pipeline_strategy}, ), - ] + ) for label, command_name, kwargs in steps: self.stdout.write(label) diff --git a/gateway/sds_gateway/api_methods/models.py b/gateway/sds_gateway/api_methods/models.py index eb290457e..dba75d136 100644 --- a/gateway/sds_gateway/api_methods/models.py +++ b/gateway/sds_gateway/api_methods/models.py @@ -933,6 +933,18 @@ def __str__(self) -> str: return self.name +class DatasetQuerySet(QuerySet["Dataset"]): + def federation_exportable(self) -> QuerySet: + return self.filter( + status=DatasetStatus.FINAL, + is_public=True, + is_deleted=False, + ) + + +DatasetManager = models.Manager.from_queryset(DatasetQuerySet) + + class Dataset(BaseModel): """ Model for datasets defined and created through the API. @@ -987,9 +999,18 @@ class Dataset(BaseModel): related_name="shared_datasets", ) + objects = DatasetManager() + def __str__(self) -> str: return self.name + def is_federation_exportable(self) -> bool: + return ( + not self.is_deleted + and self.is_public + and self.status == DatasetStatus.FINAL + ) + def _published_snapshot_from_db(self) -> tuple[str, bool] | None: if not self.pk: return None diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export.py b/gateway/sds_gateway/api_methods/tests/test_federation_export.py index 9e2503f27..34329528a 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export.py @@ -55,7 +55,10 @@ def setUp(self) -> None: status=DatasetStatus.DRAFT, keywords=None, ) - self.public_capture = CaptureFactory(owner=self.owner, is_public=True) + self.public_capture = CaptureFactory( + owner=self.owner, + datasets=[self.public_dataset], + ) self.list_datasets_url = reverse( "api:federation-export-datasets-list", ) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py index d52ef7b49..6df5d3609 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -17,6 +17,9 @@ from sds_gateway.api_methods.federation.availability import ( evaluate_federation_operational, ) +from sds_gateway.api_methods.federation.availability import ( + initialize_federation_operational_state, +) from sds_gateway.api_methods.federation.availability import ( is_client_ip_allowed_for_federation_export, ) @@ -114,6 +117,14 @@ def test_client_ip_allowlist(self) -> None: assert is_client_ip_allowed_for_federation_export(allowed) is True assert is_client_ip_allowed_for_federation_export(denied) is False + @override_settings(FEDERATION_EXPORT_ALLOWED_CIDRS=[]) + def test_empty_cidr_override_falls_back_to_private_defaults(self) -> None: + factory = RequestFactory() + loopback = factory.get("/", REMOTE_ADDR="127.0.0.1") + public = factory.get("/", REMOTE_ADDR="203.0.113.8") + assert is_client_ip_allowed_for_federation_export(loopback) is True + assert is_client_ip_allowed_for_federation_export(public) is False + @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="", @@ -141,6 +152,47 @@ def test_refresh_sets_settings_flags(self) -> None: assert reason +class TestFederationExportCidrParsing: + def test_parse_cidrs_skips_blank_tokens(self) -> None: + from config.settings.base import _parse_cidrs + + networks = _parse_cidrs(["10.0.0.0/8", "", " ", "192.168.0.0/16"]) + assert len(networks) == 2 + + def test_coerce_empty_env_tokens_to_defaults(self) -> None: + from config.settings.base import FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT + from config.settings.base import _federation_export_allowed_cidrs_from_env + + with patch( + "config.settings.base.env.list", + return_value=["", " "], + ): + networks = _federation_export_allowed_cidrs_from_env() + assert len(networks) == len(FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT) + + +class TestFederationOperationalInit: + @override_settings(FEDERATION_ENABLED=True) + @patch( + "sds_gateway.api_methods.federation.availability." + "federation_operational_db_ready", + return_value=False, + ) + @patch( + "sds_gateway.api_methods.federation.availability." + "refresh_federation_operational_state", + ) + def test_initialize_skips_db_when_tables_missing( + self, + mock_refresh: MagicMock, + _mock_db_ready: MagicMock, + ) -> None: + initialize_federation_operational_state() + mock_refresh.assert_not_called() + assert settings.FEDERATION_OPERATIONAL is False + assert settings.FEDERATION_OPERATIONAL_REASON == "database tables not ready" + + @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py b/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py new file mode 100644 index 000000000..33b2eaea8 --- /dev/null +++ b/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py @@ -0,0 +1,74 @@ +"""Tests for federation sync Token + mint endpoint.""" + +from __future__ import annotations + +import pytest +from django.test import override_settings +from django.urls import reverse +from rest_framework import status +from rest_framework.authtoken.models import Token +from rest_framework.test import APIClient + +from sds_gateway.api_methods.models import KeySources +from sds_gateway.users.backend_service_key_utils import ( + mint_federation_sync_api_key, + update_federation_sync_server_token, +) +from sds_gateway.users.models import User +from sds_gateway.users.models import UserAPIKey + +pytestmark = pytest.mark.django_db + + +@override_settings( + FEDERATION_SYNC_USER_EMAIL="federation-sync@internal.local", + FEDERATION_SYNC_SERVER_API_KEY="a" * 40, +) +class TestFederationSyncMint: + def setup_method(self) -> None: + User.objects.filter(email="federation-sync@internal.local").delete() + update_federation_sync_server_token() + + def test_mint_creates_api_key(self) -> None: + user = User.objects.get(email="federation-sync@internal.local") + raw = mint_federation_sync_api_key(user) + assert raw is not None + key = UserAPIKey.objects.get_from_key(raw) + assert key.source == KeySources.FederationSync + assert key.user_id == user.pk + + def test_mint_replaces_prior_key_like_svi(self) -> None: + user = User.objects.get(email="federation-sync@internal.local") + first = mint_federation_sync_api_key(user) + second = mint_federation_sync_api_key(user) + assert second != first + assert ( + UserAPIKey.objects.filter( + user=user, + source=KeySources.FederationSync, + ).count() + == 1 + ) + + +@override_settings( + FEDERATION_SYNC_USER_EMAIL="federation-sync@internal.local", + FEDERATION_SYNC_SERVER_API_KEY="b" * 40, +) +class TestGetFederationSyncApiKeyEndpoint: + def setup_method(self) -> None: + User.objects.filter(email="federation-sync@internal.local").delete() + update_federation_sync_server_token() + + def test_mint_via_http(self) -> None: + user = User.objects.get(email="federation-sync@internal.local") + token = Token.objects.get(user=user) + client = APIClient() + url = reverse("users:get_federation_sync_api_key") + response = client.get( + url, + {"email": user.email}, + HTTP_AUTHORIZATION=f"Token {token.key}", + ) + assert response.status_code == status.HTTP_200_OK + assert "api_key" in response.json() diff --git a/gateway/sds_gateway/users/api/views.py b/gateway/sds_gateway/users/api/views.py index 1555e5ad2..dec491b8b 100644 --- a/gateway/sds_gateway/users/api/views.py +++ b/gateway/sds_gateway/users/api/views.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from typing import cast from django.conf import settings @@ -18,9 +19,9 @@ from rest_framework.views import APIView from rest_framework.viewsets import GenericViewSet -from sds_gateway.api_methods.models import KeySources +from sds_gateway.users.backend_service_key_utils import mint_federation_sync_api_key +from sds_gateway.users.backend_service_key_utils import mint_svi_backend_api_key from sds_gateway.users.models import User -from sds_gateway.users.models import UserAPIKey from .serializers import UserSerializer @@ -49,17 +50,21 @@ def me(self, request: HttpRequest) -> Response: @extend_schema(exclude=True) -class GetAPIKeyView(APIView): +class BackendServiceMintAPIKeyView(APIView): + """Token-authenticated mint endpoint for internal backend services.""" + permission_classes = [IsAuthenticated] authentication_classes = [TokenAuthentication] + service_user_email_setting: str + unauthorized_message: str + mint_user_api_key: Callable[[User], str] def get(self, request: Request) -> Response: - """Generates an API Key for an SDS user.""" - + allowed_email = getattr(settings, self.service_user_email_setting) log.debug(f"request.user: {request.user}") - if request.user.email != settings.SVI_SERVER_EMAIL: + if request.user.email != allowed_email: return Response( - {"error": "Unauthorized. Only SVI Server can access this endpoint."}, + {"error": self.unauthorized_message}, status=status.HTTP_403_FORBIDDEN, ) @@ -70,17 +75,21 @@ def get(self, request: Request) -> Response: status=status.HTTP_400_BAD_REQUEST, ) - try: - user = get_object_or_404(User, email=email, is_approved=True) - UserAPIKey.objects.filter(user=user, source=KeySources.SVIBackend).delete() - _, raw_key = UserAPIKey.objects.create_key( - name=f"{user.email}-SVI-API-KEY", - user=user, - source=KeySources.SVIBackend, - ) - return Response({"api_key": raw_key, "email": user.email}) - except User.DoesNotExist: - return Response( - {"error": "User not found"}, - status=status.HTTP_404_NOT_FOUND, - ) + user = get_object_or_404(User, email=email, is_approved=True) + raw_key = self.mint_user_api_key(user) + return Response({"api_key": raw_key, "email": user.email}) + + +get_svi_api_key_view = BackendServiceMintAPIKeyView.as_view( + service_user_email_setting="SVI_SERVER_EMAIL", + unauthorized_message="Unauthorized. Only SVI Server can access this endpoint.", + mint_user_api_key=mint_svi_backend_api_key, +) + +get_federation_sync_api_key_view = BackendServiceMintAPIKeyView.as_view( + service_user_email_setting="FEDERATION_SYNC_USER_EMAIL", + unauthorized_message=( + "Unauthorized. Only federation sync can access this endpoint." + ), + mint_user_api_key=mint_federation_sync_api_key, +) diff --git a/gateway/sds_gateway/users/backend_service_key_utils.py b/gateway/sds_gateway/users/backend_service_key_utils.py new file mode 100644 index 000000000..76974482d --- /dev/null +++ b/gateway/sds_gateway/users/backend_service_key_utils.py @@ -0,0 +1,101 @@ +"""Internal service auth: DRF Tokens and UserAPIKey mint (SVI, federation sync).""" + +from __future__ import annotations + +import logging + +from django.conf import settings +from django.core.exceptions import ValidationError +from django.db import DatabaseError +from django.db import IntegrityError +from rest_framework.authtoken.models import Token + +from sds_gateway.api_methods.models import KeySources +from sds_gateway.users.models import User +from sds_gateway.users.models import UserAPIKey + +logger = logging.getLogger(__name__) + + +def get_or_create_service_user(email: str) -> tuple[User, bool]: + user, created = User.objects.get_or_create( + email=email, + defaults={ + "is_active": True, + "is_approved": True, + }, + ) + if created: + user.set_unusable_password() + user.save(update_fields=["password"]) + logger.info("Created service user: %s", email) + return user, created + + +def update_service_server_token(*, user_email: str, token_key: str) -> None: + """Sync DRF authtoken ``Token.key`` for an internal service user.""" + user, _created = get_or_create_service_user(user_email) + existing = Token.objects.filter(user=user) + count = existing.count() + if count: + existing.delete() + logger.info("Removed %d token(s) for service user %s", count, user_email) + Token.objects.create(user=user, key=token_key) + logger.info("Updated service token for %s", user_email) + + +def replace_user_api_key_for_source( + *, + user: User, + source: KeySources, + name: str, + description: str = "", +) -> str: + """Delete existing keys for ``(user, source)`` and mint a new Api-Key (SVI pattern).""" + UserAPIKey.objects.filter(user=user, source=source).delete() + _obj, raw_key = UserAPIKey.objects.create_key( + name=name, + user=user, + source=source, + description=description, + ) + return raw_key + + +def mint_svi_backend_api_key(user: User) -> str: + return replace_user_api_key_for_source( + user=user, + source=KeySources.SVIBackend, + name=f"{user.email}-SVI-API-KEY", + ) + + +def mint_federation_sync_api_key(user: User) -> str: + return replace_user_api_key_for_source( + user=user, + source=KeySources.FederationSync, + name="federation-sync", + description="Federation sync service (export endpoints only)", + ) + + +def update_svi_server_token() -> None: + """Update the SVI token when ``SVI_SERVER_API_KEY`` changes.""" + try: + update_service_server_token( + user_email=settings.SVI_SERVER_EMAIL, + token_key=settings.SVI_SERVER_API_KEY, + ) + except (DatabaseError, IntegrityError, ValidationError): + logger.exception("Failed to update SVI server token") + + +def update_federation_sync_server_token() -> None: + """Sync federation-sync DRF Token from ``FEDERATION_SYNC_SERVER_API_KEY``.""" + try: + update_service_server_token( + user_email=settings.FEDERATION_SYNC_USER_EMAIL, + token_key=settings.FEDERATION_SYNC_SERVER_API_KEY, + ) + except (DatabaseError, IntegrityError, ValidationError): + logger.exception("Failed to update federation sync server token") diff --git a/gateway/sds_gateway/users/management/commands/create_federation_sync_api_key.py b/gateway/sds_gateway/users/management/commands/create_federation_sync_api_key.py deleted file mode 100644 index 2c7ede4e2..000000000 --- a/gateway/sds_gateway/users/management/commands/create_federation_sync_api_key.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Create the federation sync service user and API key.""" - -from django.conf import settings -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand -from loguru import logger as log - -from sds_gateway.api_methods.models import KeySources -from sds_gateway.users.models import UserAPIKey - - -class Command(BaseCommand): - """Provision federation-sync@internal user and a FederationSync API key.""" - - help = "Create federation sync service user and API key (prints raw key once)." - - def handle(self, *args, **options) -> None: - user_model = get_user_model() - email = settings.FEDERATION_SYNC_USER_EMAIL - - user, created = user_model.objects.get_or_create( - email=email, - defaults={ - "is_active": True, - "is_approved": True, - }, - ) - if created: - user.set_unusable_password() - user.save(update_fields=["password"]) - log.info("Created federation sync user {}", email) - else: - log.info("Using existing federation sync user {}", email) - - UserAPIKey.objects.filter( - user=user, - source=KeySources.FederationSync, - ).delete() - - _obj, raw_key = UserAPIKey.objects.create_key( - name="federation-sync", - user=user, - source=KeySources.FederationSync, - description="Federation sync service (export endpoints only)", - ) - self.stdout.write(self.style.SUCCESS(f"Federation sync API key: {raw_key}")) diff --git a/gateway/sds_gateway/users/management/commands/create_test_files.py b/gateway/sds_gateway/users/management/commands/create_test_files.py deleted file mode 100644 index b85b28e0d..000000000 --- a/gateway/sds_gateway/users/management/commands/create_test_files.py +++ /dev/null @@ -1,140 +0,0 @@ -import secrets -from datetime import timedelta -from pathlib import Path - -from django.conf import settings -from django.contrib.auth import get_user_model -from django.core.management.base import BaseCommand -from django.db import DatabaseError -from django.db import IntegrityError -from django.db import transaction -from django.utils import timezone - -from sds_gateway.api_methods.models import Capture -from sds_gateway.api_methods.models import File -from sds_gateway.api_methods.utils.minio_client import get_minio_client - -User = get_user_model() # This is the only User import we need - - -class Command(BaseCommand): - help = "Create test files and captures for development" - - def handle(self, *args, **options): - try: - if not settings.DEBUG: - self.stdout.write( - self.style.ERROR( - "This command can only be run in local development environment" - ) - ) - return - user = self._get_or_create_test_user() - self._create_test_files(user) - self._create_test_captures(user) - except (DatabaseError, IntegrityError) as e: - self.stdout.write(self.style.ERROR(f"Database error: {e}")) - except OSError as e: - self.stdout.write(self.style.ERROR(f"I/O error: {e}")) - - def _get_or_create_test_user(self): - """Get or create a test user.""" - try: - return User.objects.get_or_create( - email="test@example.com", - defaults={ - "name": "Test User", - "is_staff": True, - "is_superuser": True, - }, - )[0] - except DatabaseError as e: - self.stdout.write(self.style.ERROR(f"Failed to create test user: {e}")) - raise - - def _create_test_files(self, user): - """Create test files in MinIO.""" - # Define test directories and file types - directories = [Path(f"files/{user.email}/test_dir_{i}") for i in range(3)] - - file_types = [ - ("txt", b"Test content"), - ("csv", b"id,name\n1,test"), - ("json", b'{"test": "data"}'), - ] - - # Create directories if they don't exist - for directory in directories: - directory_path = Path(str(directory)) - directory_path.mkdir(parents=True, exist_ok=True) - - # Create files - minio_client = get_minio_client() - created_files = [] - - try: - for directory in directories: - for i in range(3): - file_type, content = secrets.choice(file_types) - file_name = f"test_file_{i}.{file_type}" - file_path = directory / file_name - - # Save file - with file_path.open("wb") as f: - f.write(content) - - # Upload to MinIO - minio_client.upload_file( - str(file_path), - str(file_path), - f"text/{file_type}", - ) - - # Create file record - file_obj = File.objects.create( - name=file_name, - directory=str(directory), - media_type=f"text/{file_type}", - size=len(content), - owner=user, - ) - created_files.append(file_obj) - - self.stdout.write(self.style.SUCCESS(f"Created file: {file_path}")) - except (OSError, DatabaseError) as e: - self.stdout.write(self.style.ERROR(f"Failed to create test files: {e}")) - raise - else: - return created_files - - def _create_test_captures(self, user): - """Create test captures.""" - capture_types = ["rh", "drf"] - channels = ["ch1", "ch2", "ch3"] - scan_groups = ["group1", "group2"] - - try: - with transaction.atomic(): - for i in range(5): - capture_type = secrets.choice(capture_types) - channel = secrets.choice(channels) - scan_group = secrets.choice(scan_groups) - top_level_dir = f"captures/{user.email}/test_capture_{i}" - - capture = Capture.objects.create( - capture_type=capture_type, - top_level_dir=top_level_dir, - channel=channel, - scan_group=scan_group, - owner=user, - created_at=timezone.now() - timedelta(days=i), - ) - - # Associate some random files with the capture - associated_files = File.objects.filter(owner=user).order_by("?")[:3] - capture.files.set(associated_files) - - self.stdout.write(self.style.SUCCESS(f"Created capture: {capture}")) - - except (DatabaseError, IntegrityError) as e: - self.stdout.write(self.style.WARNING(f"Failed to create capture {i}: {e}")) diff --git a/gateway/sds_gateway/users/management/commands/init_federation_sync_token.py b/gateway/sds_gateway/users/management/commands/init_federation_sync_token.py new file mode 100644 index 000000000..b87fd320d --- /dev/null +++ b/gateway/sds_gateway/users/management/commands/init_federation_sync_token.py @@ -0,0 +1,23 @@ +"""Management command to initialize federation sync server token.""" + +from django.core.management.base import BaseCommand +from loguru import logger as log + +from sds_gateway.users.backend_service_key_utils import ( + update_federation_sync_server_token, +) + + +class Command(BaseCommand): + help = "Initialize or update the federation sync server authentication token" + + def handle(self, *args, **options) -> None: + """Initialize the federation sync server token.""" + log.info("Initializing federation sync server token...") + + try: + update_federation_sync_server_token() + log.success("Federation sync server token initialized successfully") + except Exception as e: + log.error(f"Failed to initialize federation sync server token: {e}") + raise diff --git a/gateway/sds_gateway/users/management/commands/init_svi_token.py b/gateway/sds_gateway/users/management/commands/init_svi_token.py index a28bf690a..64007130d 100644 --- a/gateway/sds_gateway/users/management/commands/init_svi_token.py +++ b/gateway/sds_gateway/users/management/commands/init_svi_token.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand from loguru import logger as log -from sds_gateway.users.svi_utils import update_svi_server_token +from sds_gateway.users.backend_service_key_utils import update_svi_server_token class Command(BaseCommand): diff --git a/gateway/sds_gateway/users/svi_utils.py b/gateway/sds_gateway/users/svi_utils.py deleted file mode 100644 index 40273a0e9..000000000 --- a/gateway/sds_gateway/users/svi_utils.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Utilities for managing SVI server authentication.""" - -from __future__ import annotations - -import logging - -from django.conf import settings -from django.core.exceptions import ValidationError -from django.db import DatabaseError -from django.db import IntegrityError -from rest_framework.authtoken.models import Token - -from sds_gateway.users.models import User - -logger = logging.getLogger(__name__) - - -def update_svi_server_token() -> None: - """Updates the SVI's token for automatic key rotation if the setting changes.""" - try: - svi_user, created = User.objects.get_or_create( - email=settings.SVI_SERVER_EMAIL, - defaults={ - "is_active": True, - "is_approved": True, - }, - ) - - if created: - logger.info("Created SVI server user: %s", settings.SVI_SERVER_EMAIL) - - __delete_svi_user_tokens(svi_user) - __create_svi_user_token(svi_user) - - except (DatabaseError, IntegrityError, ValidationError): - logger.exception("Failed to update SVI server token") - - -def __delete_svi_user_tokens(svi_user: User) -> None: - """Deletes all existing tokens for the SVI user, revoking existing keys.""" - existing_tokens = Token.objects.filter(user=svi_user) - token_count = existing_tokens.count() - if token_count > 0: - existing_tokens.delete() - logger.info("Removed %d existing token(s) for SVI server user", token_count) - - -def __create_svi_user_token(svi_user: User) -> None: - """Creates a new token for the SVI user following the active setting.""" - Token.objects.create(user=svi_user, key=settings.SVI_SERVER_API_KEY) - logger.info( - "Successfully updated SVI server token for %s", settings.SVI_SERVER_EMAIL - ) diff --git a/gateway/sds_gateway/users/tests/test_svi_utils.py b/gateway/sds_gateway/users/tests/test_svi_utils.py index 8af126081..fb66142ee 100644 --- a/gateway/sds_gateway/users/tests/test_svi_utils.py +++ b/gateway/sds_gateway/users/tests/test_svi_utils.py @@ -6,7 +6,7 @@ from rest_framework.authtoken.models import Token from sds_gateway.users.models import User -from sds_gateway.users.svi_utils import update_svi_server_token +from sds_gateway.users.backend_service_key_utils import update_svi_server_token @pytest.mark.django_db diff --git a/gateway/sds_gateway/users/urls.py b/gateway/sds_gateway/users/urls.py index 635241e9a..4443b4c40 100644 --- a/gateway/sds_gateway/users/urls.py +++ b/gateway/sds_gateway/users/urls.py @@ -1,7 +1,8 @@ from django.urls import path from django.views.generic import RedirectView -from .api.views import GetAPIKeyView +from .api.views import get_federation_sync_api_key_view +from .api.views import get_svi_api_key_view from .views import CheckFileExistsView from .views import FileContentView from .views import FileDownloadView @@ -129,7 +130,12 @@ path("upload-files/", UploadCaptureView.as_view(), name="upload_files"), path("check-file-exists/", CheckFileExistsView.as_view(), name="check_file_exists"), # Used by SVI Server to get API key for a user - path("get-svi-api-key/", GetAPIKeyView.as_view(), name="get_svi_api_key"), + path("get-svi-api-key/", get_svi_api_key_view, name="get_svi_api_key"), + path( + "get-federation-sync-api-key/", + get_federation_sync_api_key_view, + name="get_federation_sync_api_key", + ), path("revoke-api-key/", revoke_api_key_view, name="revoke_api_key"), path( "generate-api-key-form/", From e2635a1eb2a18e71236d2c412d609a00930494f3 Mon Sep 17 00:00:00 2001 From: klpoland Date: Thu, 9 Jul 2026 16:07:38 -0400 Subject: [PATCH 11/17] generate drf token key with generate-secrets into shared env file between gateway and federation --- gateway/.envs/example/django.env | 8 ++--- gateway/config/settings/base.py | 7 ++-- gateway/scripts/generate-secrets.sh | 35 +++++++++++++++++++ .../tests/test_federation_sync_api_key.py | 25 ++++++------- .../users/backend_service_key_utils.py | 16 ++++++--- .../commands/init_federation_sync_token.py | 6 ++-- .../tests/test_federation_sync_drf_init.py | 28 +++++++++++++++ 7 files changed, 92 insertions(+), 33 deletions(-) create mode 100644 gateway/sds_gateway/users/tests/test_federation_sync_drf_init.py diff --git a/gateway/.envs/example/django.env b/gateway/.envs/example/django.env index f4f4607a6..904620642 100644 --- a/gateway/.envs/example/django.env +++ b/gateway/.envs/example/django.env @@ -10,9 +10,9 @@ API_KEY= # FEDERATION # ------------------------------------------------------------------------------ -# Peer sync: shared FEDERATION_SYNC_SERVER_API_KEY in django.env + federation sync.env; -# prepare_gateway runs init_federation_sync_token. Sync startup calls -# GET /users/get-federation-sync-api-key/ (Token auth) and writes FEDERATION_GATEWAY_API_KEY locally. +# Load ../federation/.envs//federation-shared.env on gateway + sync (generate-secrets). +# prepare_gateway runs init_federation_sync_token (DB Token from FEDERATION_SYNC_DRF_TOKEN). +# Sync startup mints FEDERATION_SYNC_SERVER_API_KEY via get-federation-sync-api-key. # Set FEDERATION_SITE_NAME (e.g. crc) when enabling federation; use SDS_SITE_FQDN for the public host (RFC [site].fqdn). # Master switch for export APIs and Redis federation events. # FEDERATION_ENABLED=true @@ -23,8 +23,6 @@ API_KEY= # Health probe target (federation-sync service). # FEDERATION_SYNC_HEALTH_URL=http://federation-sync:8000/sync/health # FEDERATION_SYNC_USER_EMAIL=federation-sync@internal.local -# Bootstrap Token (shared with federation sync.env); not the export Api-Key. -# FEDERATION_SYNC_SERVER_API_KEY= # Comma-separated CIDRs allowed to call export (default: private Docker ranges). # FEDERATION_EXPORT_ALLOWED_CIDRS= diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index 516c49ed9..9914549e0 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -767,11 +767,8 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: "FEDERATION_SYNC_USER_EMAIL", default="federation-sync@internal.local", ) -# Shared with federation-sync (Authorization: Token …) to call get-federation-sync-api-key. -FEDERATION_SYNC_SERVER_API_KEY: str = env.str( - "FEDERATION_SYNC_SERVER_API_KEY", - default=__get_random_token(TOKEN_LENGTH), -) +# From federation-shared.env (compose env_file); used by init_federation_sync_token only. +FEDERATION_SYNC_DRF_TOKEN: str = env.str("FEDERATION_SYNC_DRF_TOKEN", default="") FEDERATION_SYNC_HEALTH_URL: str = env.str( "FEDERATION_SYNC_HEALTH_URL", default="http://federation-sync:8000/sync/health", diff --git a/gateway/scripts/generate-secrets.sh b/gateway/scripts/generate-secrets.sh index ea6e180c7..1602dc39a 100755 --- a/gateway/scripts/generate-secrets.sh +++ b/gateway/scripts/generate-secrets.sh @@ -3,6 +3,7 @@ set -Eeuo pipefail SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) GATEWAY_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) +FEDERATION_ROOT=$(cd "${GATEWAY_ROOT}/../federation" && pwd) SFS_ROOT=$(cd "${GATEWAY_ROOT}/../seaweedfs" && pwd) EXAMPLE_DIR="${GATEWAY_ROOT}/.envs/example" @@ -16,6 +17,20 @@ PRIMARY_S3_ENDPOINT_URL="" SECONDARY_ACCESS_KEY_ID="" SECONDARY_SECRET_ACCESS_KEY="" +FEDERATION_SYNC_DRF_TOKEN="" + +function ensure_federation_sync_drf_token() { + local env_type="$1" + if [[ -n "${FEDERATION_SYNC_DRF_TOKEN}" ]]; then + return 0 + fi + if [[ "${env_type}" == "ci" ]]; then + FEDERATION_SYNC_DRF_TOKEN="ci-federation-sync-drf-01234567890123456" + else + FEDERATION_SYNC_DRF_TOKEN=$(generate_secret 40) + fi +} + function usage() { cat < @@ -36,6 +51,7 @@ EXAMPLES: NOTES: - Generated files are placed in .envs// directory + - Federation federation-shared.env → ../federation/.envs//federation-shared.env - Example templates are read from .envs/example/ - Secrets are randomly generated using OpenSSL - CI environment uses insecure but deterministic values for ephemeral usage @@ -167,6 +183,11 @@ function process_env_file() { # set WEB_CONCURRENCY based on CPU cores (applies to all environments) content="${content//WEB_CONCURRENCY=4/WEB_CONCURRENCY=${web_concurrency}}" + if [[ "${filename}" == "federation-shared.env" ]]; then + ensure_federation_sync_drf_token "${env_type}" + content="${content//FEDERATION_SYNC_DRF_TOKEN=/FEDERATION_SYNC_DRF_TOKEN=${FEDERATION_SYNC_DRF_TOKEN}}" + fi + if [[ "${filename}" == "storage.env" ]]; then # PRIMARY vars content="${content//PRIMARY_ACCESS_KEY_ID=admin/PRIMARY_ACCESS_KEY_ID=${PRIMARY_ACCESS_KEY_ID}}" @@ -243,6 +264,7 @@ function set_permissions() { env_dirs=( "${GATEWAY_ROOT}/.envs" "${SFS_ROOT}/.envs" + "${FEDERATION_ROOT}/.envs" ) for dir in "${env_dirs[@]}"; do if [ -d "${dir}" ]; then @@ -322,10 +344,23 @@ function main() { continue fi + # federation-shared.env is written under ../federation/.envs/ + if [[ "${filename}" == "federation-shared.env" ]]; then + continue + fi + local output="${target_dir_gwy}/${filename}" process_env_file "${template}" "${output}" "${env_type}" "${force}" done + if [[ "${env_type}" != "ci" ]]; then + process_env_file \ + "${EXAMPLE_DIR}/federation-shared.env" \ + "${FEDERATION_ROOT}/.envs/${env_type}/federation-shared.env" \ + "${env_type}" \ + "${force}" + fi + set_permissions echo "" diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py b/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py index 33b2eaea8..395968fc1 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py @@ -3,31 +3,27 @@ from __future__ import annotations import pytest -from django.test import override_settings from django.urls import reverse from rest_framework import status from rest_framework.authtoken.models import Token from rest_framework.test import APIClient from sds_gateway.api_methods.models import KeySources -from sds_gateway.users.backend_service_key_utils import ( - mint_federation_sync_api_key, - update_federation_sync_server_token, -) +from sds_gateway.users.backend_service_key_utils import mint_federation_sync_api_key +from sds_gateway.users.backend_service_key_utils import update_service_server_token from sds_gateway.users.models import User from sds_gateway.users.models import UserAPIKey pytestmark = pytest.mark.django_db -@override_settings( - FEDERATION_SYNC_USER_EMAIL="federation-sync@internal.local", - FEDERATION_SYNC_SERVER_API_KEY="a" * 40, -) class TestFederationSyncMint: def setup_method(self) -> None: User.objects.filter(email="federation-sync@internal.local").delete() - update_federation_sync_server_token() + update_service_server_token( + user_email="federation-sync@internal.local", + token_key="a" * 40, + ) def test_mint_creates_api_key(self) -> None: user = User.objects.get(email="federation-sync@internal.local") @@ -51,14 +47,13 @@ def test_mint_replaces_prior_key_like_svi(self) -> None: ) -@override_settings( - FEDERATION_SYNC_USER_EMAIL="federation-sync@internal.local", - FEDERATION_SYNC_SERVER_API_KEY="b" * 40, -) class TestGetFederationSyncApiKeyEndpoint: def setup_method(self) -> None: User.objects.filter(email="federation-sync@internal.local").delete() - update_federation_sync_server_token() + update_service_server_token( + user_email="federation-sync@internal.local", + token_key="b" * 40, + ) def test_mint_via_http(self) -> None: user = User.objects.get(email="federation-sync@internal.local") diff --git a/gateway/sds_gateway/users/backend_service_key_utils.py b/gateway/sds_gateway/users/backend_service_key_utils.py index 76974482d..8501c119a 100644 --- a/gateway/sds_gateway/users/backend_service_key_utils.py +++ b/gateway/sds_gateway/users/backend_service_key_utils.py @@ -90,12 +90,20 @@ def update_svi_server_token() -> None: logger.exception("Failed to update SVI server token") -def update_federation_sync_server_token() -> None: - """Sync federation-sync DRF Token from ``FEDERATION_SYNC_SERVER_API_KEY``.""" +def update_federation_sync_drf_token() -> None: + """Sync federation-sync DRF ``Token`` from ``FEDERATION_SYNC_DRF_TOKEN`` (shared env).""" + token_key = settings.FEDERATION_SYNC_DRF_TOKEN + if len(token_key) != settings.TOKEN_LENGTH: + msg = ( + f"FEDERATION_SYNC_DRF_TOKEN must be {settings.TOKEN_LENGTH} characters " + f"(load federation-shared.env on the gateway container)" + ) + raise ValueError(msg) try: update_service_server_token( user_email=settings.FEDERATION_SYNC_USER_EMAIL, - token_key=settings.FEDERATION_SYNC_SERVER_API_KEY, + token_key=token_key, ) except (DatabaseError, IntegrityError, ValidationError): - logger.exception("Failed to update federation sync server token") + logger.exception("Failed to update federation sync DRF token") + raise diff --git a/gateway/sds_gateway/users/management/commands/init_federation_sync_token.py b/gateway/sds_gateway/users/management/commands/init_federation_sync_token.py index b87fd320d..48e5d0e7b 100644 --- a/gateway/sds_gateway/users/management/commands/init_federation_sync_token.py +++ b/gateway/sds_gateway/users/management/commands/init_federation_sync_token.py @@ -3,9 +3,7 @@ from django.core.management.base import BaseCommand from loguru import logger as log -from sds_gateway.users.backend_service_key_utils import ( - update_federation_sync_server_token, -) +from sds_gateway.users.backend_service_key_utils import update_federation_sync_drf_token class Command(BaseCommand): @@ -16,7 +14,7 @@ def handle(self, *args, **options) -> None: log.info("Initializing federation sync server token...") try: - update_federation_sync_server_token() + update_federation_sync_drf_token() log.success("Federation sync server token initialized successfully") except Exception as e: log.error(f"Failed to initialize federation sync server token: {e}") diff --git a/gateway/sds_gateway/users/tests/test_federation_sync_drf_init.py b/gateway/sds_gateway/users/tests/test_federation_sync_drf_init.py new file mode 100644 index 000000000..4fa39578c --- /dev/null +++ b/gateway/sds_gateway/users/tests/test_federation_sync_drf_init.py @@ -0,0 +1,28 @@ +"""Tests for federation sync DRF token init from shared env.""" + +from __future__ import annotations + +import pytest +from django.test import override_settings +from rest_framework.authtoken.models import Token + +from sds_gateway.users.backend_service_key_utils import update_federation_sync_drf_token +from sds_gateway.users.models import User + +pytestmark = pytest.mark.django_db + + +@override_settings(FEDERATION_SYNC_DRF_TOKEN="a" * 40) +def test_update_federation_sync_drf_token_from_settings() -> None: + User.objects.filter(email="federation-sync@internal.local").delete() + + update_federation_sync_drf_token() + + user = User.objects.get(email="federation-sync@internal.local") + assert Token.objects.get(user=user).key == "a" * 40 + + +@override_settings(FEDERATION_SYNC_DRF_TOKEN="") +def test_update_requires_token_length() -> None: + with pytest.raises(ValueError, match="FEDERATION_SYNC_DRF_TOKEN"): + update_federation_sync_drf_token() From ee4b2cac6dccc7d5401fa96e6ba59a5d91e45961 Mon Sep 17 00:00:00 2001 From: klpoland Date: Thu, 9 Jul 2026 16:19:40 -0400 Subject: [PATCH 12/17] shared env on repo root --- .gitignore | 1 + federation-shared.example.env | 7 +++++ gateway/.envs/example/django.env | 2 +- gateway/compose.local.yaml | 1 + gateway/scripts/generate-secrets.sh | 41 ++++++++++++++++++----------- 5 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 federation-shared.example.env diff --git a/.gitignore b/.gitignore index 5cea5cf77..4c5655055 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ **/agents.md agents.md htmlcov/ +federation-shared.env \ No newline at end of file diff --git a/federation-shared.example.env b/federation-shared.example.env new file mode 100644 index 000000000..0578122de --- /dev/null +++ b/federation-shared.example.env @@ -0,0 +1,7 @@ +# Shared by gateway + federation sync (repo root; gateway/scripts/generate-secrets.sh). +# Compose: env_file → ../../federation-shared.env (gateway) or ../federation-shared.env (federation). +# FEDERATION_SYNC_DRF_TOKEN: bootstrap Token for get-federation-sync-api-key. +# FEDERATION_SYNC_SERVER_API_KEY: export Api-Key (sync service writes after mint). + +FEDERATION_SYNC_DRF_TOKEN= +FEDERATION_SYNC_SERVER_API_KEY= diff --git a/gateway/.envs/example/django.env b/gateway/.envs/example/django.env index 904620642..500d77454 100644 --- a/gateway/.envs/example/django.env +++ b/gateway/.envs/example/django.env @@ -10,7 +10,7 @@ API_KEY= # FEDERATION # ------------------------------------------------------------------------------ -# Load ../federation/.envs//federation-shared.env on gateway + sync (generate-secrets). +# Load federation-shared.env from repo root on gateway + sync compose stacks (generate-secrets). # prepare_gateway runs init_federation_sync_token (DB Token from FEDERATION_SYNC_DRF_TOKEN). # Sync startup mints FEDERATION_SYNC_SERVER_API_KEY via get-federation-sync-api-key. # Set FEDERATION_SITE_NAME (e.g. crc) when enabling federation; use SDS_SITE_FQDN for the public host (RFC [site].fqdn). diff --git a/gateway/compose.local.yaml b/gateway/compose.local.yaml index fd9343a5d..0f0406aa9 100644 --- a/gateway/compose.local.yaml +++ b/gateway/compose.local.yaml @@ -83,6 +83,7 @@ services: - ./.envs/local/storage.env - ./.envs/local/postgres.env - ./.envs/local/opensearch.env + - ../../federation-shared.env environment: - SKIP_POSTGRES_WAIT=1 # remember /entrypoint runs first diff --git a/gateway/scripts/generate-secrets.sh b/gateway/scripts/generate-secrets.sh index 1602dc39a..cdb74318f 100755 --- a/gateway/scripts/generate-secrets.sh +++ b/gateway/scripts/generate-secrets.sh @@ -3,9 +3,12 @@ set -Eeuo pipefail SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) GATEWAY_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) -FEDERATION_ROOT=$(cd "${GATEWAY_ROOT}/../federation" && pwd) -SFS_ROOT=$(cd "${GATEWAY_ROOT}/../seaweedfs" && pwd) +SDS_ROOT=$(cd "${GATEWAY_ROOT}/.." && pwd) +FEDERATION_ROOT=$(cd "${SDS_ROOT}/federation" && pwd) +SFS_ROOT=$(cd "${SDS_ROOT}/seaweedfs" && pwd) EXAMPLE_DIR="${GATEWAY_ROOT}/.envs/example" +FEDERATION_SHARED_TEMPLATE="${SDS_ROOT}/federation-shared.example.env" +FEDERATION_SHARED_OUTPUT="${SDS_ROOT}/federation-shared.env" # PRIMARY (RustFS or SeaweedFS) PRIMARY_ACCESS_KEY_ID="" @@ -50,8 +53,8 @@ EXAMPLES: ${0} production # Generate production env files NOTES: - - Generated files are placed in .envs// directory - - Federation federation-shared.env → ../federation/.envs//federation-shared.env + - Generated gateway files are placed in gateway/.envs// + - federation-shared.env is written to the repo root (from federation-shared.example.env) - Example templates are read from .envs/example/ - Secrets are randomly generated using OpenSSL - CI environment uses insecure but deterministic values for ephemeral usage @@ -136,7 +139,7 @@ function process_env_file() { return 0 fi - echo " ✓ Generating ${output}" + echo " ✓ Generating ${output} (from ${template})" local content content=$(cat "${template}") @@ -183,7 +186,7 @@ function process_env_file() { # set WEB_CONCURRENCY based on CPU cores (applies to all environments) content="${content//WEB_CONCURRENCY=4/WEB_CONCURRENCY=${web_concurrency}}" - if [[ "${filename}" == "federation-shared.env" ]]; then + if [[ "${filename}" == "federation-shared.example.env" ]]; then ensure_federation_sync_drf_token "${env_type}" content="${content//FEDERATION_SYNC_DRF_TOKEN=/FEDERATION_SYNC_DRF_TOKEN=${FEDERATION_SYNC_DRF_TOKEN}}" fi @@ -253,7 +256,11 @@ function process_env_file() { # content="${content//RUSTFS_SECRET_ACCESS_KEY=/RUSTFS_SECRET_ACCESS_KEY=${SECONDARY_SECRET_ACCESS_KEY}}" fi - # write to output + # write to output (never overwrite *.example.env templates) + if [[ "${output}" == *".example.env" ]]; then + echo "ERROR: refused to write secrets to example template: ${output}" >&2 + exit 1 + fi mkdir -p "$(dirname "${output}")" echo "${content}" >"${output}" chmod 600 "${output}" @@ -271,6 +278,9 @@ function set_permissions() { find "${dir}" -type f -name "*.env" -exec chmod --changes 600 {} \; fi done + if [[ -f "${FEDERATION_SHARED_OUTPUT}" ]]; then + chmod --changes 600 "${FEDERATION_SHARED_OUTPUT}" + fi } function main() { @@ -344,27 +354,28 @@ function main() { continue fi - # federation-shared.env is written under ../federation/.envs/ - if [[ "${filename}" == "federation-shared.env" ]]; then - continue - fi - local output="${target_dir_gwy}/${filename}" process_env_file "${template}" "${output}" "${env_type}" "${force}" done - if [[ "${env_type}" != "ci" ]]; then + if [[ -f "${FEDERATION_SHARED_TEMPLATE}" ]]; then + # Read federation-shared.example.env → write repo-root federation-shared.env only process_env_file \ - "${EXAMPLE_DIR}/federation-shared.env" \ - "${FEDERATION_ROOT}/.envs/${env_type}/federation-shared.env" \ + "${FEDERATION_SHARED_TEMPLATE}" \ + "${FEDERATION_SHARED_OUTPUT}" \ "${env_type}" \ "${force}" + else + echo " ⏭ ${FEDERATION_SHARED_TEMPLATE} not found (skipping federation-shared.env)" fi set_permissions echo "" echo "✅ Secrets generated successfully in ${target_dir_gwy}/" + if [[ -f "${FEDERATION_SHARED_OUTPUT}" ]]; then + echo " Federation shared env: ${FEDERATION_SHARED_OUTPUT}" + fi echo "" echo "Next steps:" if [[ "${env_type}" == "ci" ]]; then From 08158b550e2a37c32fea1c3ca7253ae755a04b9f Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 10 Jul 2026 11:19:39 -0400 Subject: [PATCH 13/17] remove event type logic, refactor capture/dataset (re)index protocols, make sure serializers have fields to pre-filter fed searches on gateway --- .../federation/compile_federated_data.py | 16 -- .../api_methods/federation/events.py | 11 - .../api_methods/federation/fed_index.py | 21 +- .../api_methods/federation/reindex.py | 119 ++++++++++ .../api_methods/federation/signals.py | 209 ++--------------- .../serializers/capture_serializers.py | 18 +- .../serializers/dataset_serializers.py | 2 + .../tests/test_federation_events.py | 24 +- .../tests/test_federation_signals.py | 222 ++++++++++-------- .../api_methods/utils/asset_access_control.py | 11 + 10 files changed, 292 insertions(+), 361 deletions(-) create mode 100644 gateway/sds_gateway/api_methods/federation/reindex.py diff --git a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py index ca68689cf..4d47d5c28 100644 --- a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py +++ b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py @@ -109,22 +109,6 @@ def fed_doc_exists( ) is not None -def fed_doc_is_tombstoned( - *, - asset_type: ItemType, - site_name: str, - instance: Dataset | Capture, -) -> bool: - source = get_federated_export_doc_by_uuid( - asset_type=asset_type, - asset_uuid=instance.uuid, - site_name=site_name, - ) - if source is None: - return False - return source.get("is_federated_deleted") is True - - def public_datasets_queryset() -> QuerySet[Dataset]: return ( Dataset.objects.federation_exportable() diff --git a/gateway/sds_gateway/api_methods/federation/events.py b/gateway/sds_gateway/api_methods/federation/events.py index 5f17e2033..2e7d5760d 100644 --- a/gateway/sds_gateway/api_methods/federation/events.py +++ b/gateway/sds_gateway/api_methods/federation/events.py @@ -5,7 +5,6 @@ import json from datetime import UTC from datetime import datetime -from enum import StrEnum from typing import TYPE_CHECKING from typing import Any @@ -20,18 +19,9 @@ from sds_gateway.api_methods.models import ItemType -class FederationEventType(StrEnum): - CREATED = "created" - UPDATED = "updated" - DELETED = "deleted" - - def __str__(self) -> str: - return self.value - def publish_federation_event( *, - event_type: FederationEventType, item_type: ItemType, uuid: UUID, timestamp: datetime | None = None, @@ -42,7 +32,6 @@ def publish_federation_event( return channel = settings.FEDERATION_EVENTS_CHANNEL payload: dict[str, Any] = { - "event_type": str(event_type), "item_type": item_type.value, "uuid": str(uuid), "timestamp": (timestamp or datetime.now(UTC)).isoformat(), diff --git a/gateway/sds_gateway/api_methods/federation/fed_index.py b/gateway/sds_gateway/api_methods/federation/fed_index.py index b04fc9e95..e255d6773 100644 --- a/gateway/sds_gateway/api_methods/federation/fed_index.py +++ b/gateway/sds_gateway/api_methods/federation/fed_index.py @@ -9,7 +9,6 @@ from opensearchpy import OpenSearch from sds_gateway.api_methods.models import ItemType -_EVENT_DELETED = "deleted" FED_DATASETS_INDEX = "fed-datasets" FED_CAPTURES_INDEX = "fed-captures" @@ -36,7 +35,6 @@ def __init__(self, client: OpenSearch) -> None: def apply_local_event( self, *, - event_type: str, event_at: datetime, site_name: str, item_type: ItemType, @@ -45,26 +43,9 @@ def apply_local_event( ) -> None: index_name = index_for_item_type(item_type) doc_id = federated_doc_id(site_name, uuid) - event_iso = event_at.isoformat() - - if event_type == _EVENT_DELETED: - tombstone = { - "is_federated_deleted": True, - "federation_event_at": event_iso, - } - upsert_body = {**body, **tombstone} - self._client.update( - index=index_name, - id=doc_id, - body={"doc": tombstone, "upsert": upsert_body}, - refresh="wait_for", - ) - return - doc = { **body, - "is_federated_deleted": False, - "federation_event_at": event_iso, + "federation_event_at": event_at.isoformat(), } self._client.index( index=index_name, diff --git a/gateway/sds_gateway/api_methods/federation/reindex.py b/gateway/sds_gateway/api_methods/federation/reindex.py new file mode 100644 index 000000000..204ef3324 --- /dev/null +++ b/gateway/sds_gateway/api_methods/federation/reindex.py @@ -0,0 +1,119 @@ +"""Reindex federation export documents and notify sync.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from loguru import logger as log + +from sds_gateway.api_methods.federation.availability import is_federation_operational +from sds_gateway.api_methods.federation.compile_federated_data import ( + capture_in_published_dataset, +) +from sds_gateway.api_methods.federation.compile_federated_data import compile_federated_doc +from sds_gateway.api_methods.federation.compile_federated_data import fed_doc_exists +from sds_gateway.api_methods.federation.compile_federated_data import ( + get_federated_export_doc_by_uuid, +) +from sds_gateway.api_methods.federation.compile_federated_data import federation_site_name +from sds_gateway.api_methods.federation.events import publish_federation_event +from sds_gateway.api_methods.federation.fed_index import LocalFederatedIndexer +from sds_gateway.api_methods.models import Capture +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import ItemType +from sds_gateway.api_methods.utils.opensearch_client import get_opensearch_client + + +def reindex_federated_asset( + instance: Dataset | Capture, + item_type: ItemType, +) -> None: + if not is_federation_operational(): + log.debug("Federation indexing disabled; skipping reindex") + return + + site_name = federation_site_name() + timestamp = instance.updated_at + body = compile_federated_doc(instance) + + try: + LocalFederatedIndexer(get_opensearch_client()).apply_local_event( + event_at=timestamp, + site_name=site_name, + item_type=item_type, + uuid=instance.uuid, + body=body, + ) + except Exception as exc: # noqa: BLE001 + log.warning("Federation OpenSearch index failed: {}", exc) + return + + publish_federation_event( + item_type=item_type, + uuid=instance.uuid, + timestamp=timestamp, + ) + + +def reindex_captures_after_dataset_unlink(capture_pks: Iterable[int]) -> None: + pks = list(capture_pks) + if not pks: + return + for capture in Capture.objects.filter(pk__in=pks): + if capture_needs_federation_reindex(capture): + reindex_federated_asset(capture, ItemType.CAPTURE) + + +def _fed_doc_shows_is_deleted( + instance: Dataset | Capture, + item_type: ItemType, +) -> bool: + site_name = federation_site_name() + if not site_name: + return False + source = get_federated_export_doc_by_uuid( + asset_type=item_type, + asset_uuid=instance.uuid, + site_name=site_name, + ) + if source is None: + return False + return source.get("is_deleted") is True + + +def dataset_needs_federation_reindex(dataset: Dataset) -> bool: + if dataset.is_deleted: + return not _fed_doc_shows_is_deleted(dataset, ItemType.DATASET) and fed_doc_exists( + asset_type=ItemType.DATASET, + site_name=federation_site_name(), + instance=dataset, + ) + if dataset.is_federation_exportable(): + return True + site_name = federation_site_name() + if not site_name: + return False + return fed_doc_exists( + asset_type=ItemType.DATASET, + site_name=site_name, + instance=dataset, + ) + + +def capture_needs_federation_reindex(capture: Capture) -> bool: + if capture.is_deleted: + return not _fed_doc_shows_is_deleted(capture, ItemType.CAPTURE) and fed_doc_exists( + asset_type=ItemType.CAPTURE, + site_name=federation_site_name(), + instance=capture, + ) + if capture_in_published_dataset(capture): + return True + site_name = federation_site_name() + if not site_name: + return False + return fed_doc_exists( + asset_type=ItemType.CAPTURE, + site_name=site_name, + instance=capture, + ) diff --git a/gateway/sds_gateway/api_methods/federation/signals.py b/gateway/sds_gateway/api_methods/federation/signals.py index 77b9471b8..6cdc0a530 100644 --- a/gateway/sds_gateway/api_methods/federation/signals.py +++ b/gateway/sds_gateway/api_methods/federation/signals.py @@ -2,209 +2,40 @@ from __future__ import annotations -from django.conf import settings from django.db.models.signals import post_save from django.dispatch import receiver -from loguru import logger as log -from sds_gateway.api_methods.federation.compile_federated_data import ( - capture_in_other_published_datasets, -) -from sds_gateway.api_methods.federation.compile_federated_data import ( - capture_in_published_dataset, -) -from sds_gateway.api_methods.federation.compile_federated_data import ( - compile_federated_doc, -) -from sds_gateway.api_methods.federation.compile_federated_data import ( - fed_doc_exists, -) -from sds_gateway.api_methods.federation.compile_federated_data import ( - fed_doc_is_tombstoned, -) -from sds_gateway.api_methods.federation.compile_federated_data import ( - federation_site_name, -) -from sds_gateway.api_methods.federation.events import FederationEventType -from sds_gateway.api_methods.federation.events import publish_federation_event -from sds_gateway.api_methods.federation.fed_index import LocalFederatedIndexer +from sds_gateway.api_methods.federation.availability import is_federation_operational +from sds_gateway.api_methods.federation.reindex import capture_needs_federation_reindex +from sds_gateway.api_methods.federation.reindex import dataset_needs_federation_reindex +from sds_gateway.api_methods.federation.reindex import reindex_federated_asset from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset from sds_gateway.api_methods.models import ItemType -from sds_gateway.api_methods.utils.opensearch_client import get_opensearch_client -def _federation_indexing_enabled() -> bool: - if not getattr(settings, "FEDERATION_ENABLED", False): - return False - return bool(federation_site_name()) - - -def _event_type(*, created: bool, deleted: bool) -> FederationEventType: - if deleted: - return FederationEventType.DELETED - if created: - return FederationEventType.CREATED - return FederationEventType.UPDATED - - -def _federation_created_flag( - instance: Dataset | Capture, - item_type: ItemType, -) -> bool: - site_name = federation_site_name() - if not fed_doc_exists( - asset_type=item_type, - site_name=site_name, - instance=instance, - ): - return True - return fed_doc_is_tombstoned( - asset_type=item_type, - site_name=site_name, - instance=instance, - ) - - -def _ignore_stale_deleted_update( - instance: Dataset | Capture, - item_type: ItemType, -) -> bool: - if not instance.is_deleted: - return False - site_name = federation_site_name() - if not fed_doc_exists( - asset_type=item_type, - site_name=site_name, - instance=instance, - ): - return True - return fed_doc_is_tombstoned( - asset_type=item_type, - site_name=site_name, - instance=instance, - ) - - -def _index_and_notify( - *, - instance: Dataset | Capture, - item_type: ItemType, - created: bool, - deleted: bool, -) -> None: - if not _federation_indexing_enabled(): - log.debug("Federation indexing disabled; skipping signal") - return - - site_name = federation_site_name() - event_type = _event_type(created=created, deleted=deleted) - timestamp = instance.updated_at - body = compile_federated_doc(instance) - - try: - LocalFederatedIndexer(get_opensearch_client()).apply_local_event( - event_type=str(event_type), - event_at=timestamp, - site_name=site_name, - item_type=item_type, - uuid=instance.uuid, - body=body, - ) - except Exception as exc: # noqa: BLE001 - log.warning("Federation OpenSearch index failed: {}", exc) - return - - publish_federation_event( - event_type=event_type, - item_type=item_type, - uuid=instance.uuid, - timestamp=timestamp, - ) - - -def _tombstone_federated_if_exists( - instance: Dataset | Capture, - item_type: ItemType, -) -> None: - site_name = federation_site_name() - if not fed_doc_exists( - asset_type=item_type, - site_name=site_name, - instance=instance, - ): - return - if fed_doc_is_tombstoned( - asset_type=item_type, - site_name=site_name, - instance=instance, - ): - return - _index_and_notify( - instance=instance, - item_type=item_type, - created=False, - deleted=True, - ) - - -def _upsert_federated(instance: Dataset | Capture, item_type: ItemType) -> None: - created = _federation_created_flag(instance, item_type) - _index_and_notify( - instance=instance, - item_type=item_type, - created=created, - deleted=False, - ) - - -def _maybe_tombstone_capture_after_dataset_change( - capture: Capture, - *, - dataset: Dataset, -) -> None: - if _ignore_stale_deleted_update(capture, ItemType.CAPTURE): - return - if capture_in_other_published_datasets( - capture, - exclude_dataset_id=dataset.uuid, - ): - return - _tombstone_federated_if_exists(capture, ItemType.CAPTURE) - - -def _sync_published_captures_for_dataset(dataset: Dataset) -> None: +def _reindex_dataset_captures(dataset: Dataset) -> None: for capture in dataset.captures.filter(is_deleted=False): - if not capture_in_published_dataset(capture): - continue - if _ignore_stale_deleted_update(capture, ItemType.CAPTURE): - continue - _upsert_federated(capture, ItemType.CAPTURE) + if capture_needs_federation_reindex(capture): + reindex_federated_asset(capture, ItemType.CAPTURE) @receiver(post_save, sender=Dataset) def federation_dataset_changed( sender: type[Dataset], instance: Dataset, - created: bool, # noqa: FBT001 + created: bool, # noqa: FBT001, ARG001 **kwargs, ) -> None: - if not _federation_indexing_enabled(): - return - if _ignore_stale_deleted_update(instance, ItemType.DATASET): + if not is_federation_operational(): return - - if instance.is_deleted: - _tombstone_federated_if_exists(instance, ItemType.DATASET) - for capture in instance.captures.all(): - _maybe_tombstone_capture_after_dataset_change(capture, dataset=instance) + if not dataset_needs_federation_reindex(instance): return - if not instance.is_federation_exportable(): + reindex_federated_asset(instance, ItemType.DATASET) + if instance.is_deleted or not instance.is_federation_exportable(): return - - _upsert_federated(instance, ItemType.DATASET) - _sync_published_captures_for_dataset(instance) + _reindex_dataset_captures(instance) @receiver(post_save, sender=Capture) @@ -213,16 +44,8 @@ def federation_capture_changed( instance: Capture, **kwargs, ) -> None: - if not _federation_indexing_enabled(): + if not is_federation_operational(): return - if _ignore_stale_deleted_update(instance, ItemType.CAPTURE): + if not capture_needs_federation_reindex(instance): return - - if instance.is_deleted: - _tombstone_federated_if_exists(instance, ItemType.CAPTURE) - return - - if not capture_in_published_dataset(instance): - return - - _upsert_federated(instance, ItemType.CAPTURE) + reindex_federated_asset(instance, ItemType.CAPTURE) diff --git a/gateway/sds_gateway/api_methods/serializers/capture_serializers.py b/gateway/sds_gateway/api_methods/serializers/capture_serializers.py index 168178013..209763f0c 100644 --- a/gateway/sds_gateway/api_methods/serializers/capture_serializers.py +++ b/gateway/sds_gateway/api_methods/serializers/capture_serializers.py @@ -892,7 +892,7 @@ class CaptureFederationSerializer(serializers.ModelSerializer[Capture]): file_count = serializers.SerializerMethodField() size = serializers.SerializerMethodField() capture_props = serializers.SerializerMethodField() - dataset_ids = serializers.SerializerMethodField() + public_dataset_ids = serializers.SerializerMethodField() created_at = serializers.DateTimeField( format="%Y-%m-%d %H:%M:%S%z", read_only=True, @@ -901,6 +901,10 @@ class CaptureFederationSerializer(serializers.ModelSerializer[Capture]): format="%Y-%m-%d %H:%M:%S%z", read_only=True, ) + deleted_at = serializers.DateTimeField( + format="%Y-%m-%d %H:%M:%S%z", + read_only=True, + ) class Meta: model = Capture @@ -911,13 +915,15 @@ class Meta: "channel", "scan_group", "top_level_dir", - "created_at", - "updated_at", "site_name", "file_count", "size", "capture_props", - "dataset_ids", + "public_dataset_ids", + "created_at", + "updated_at", + "is_deleted", + "deleted_at", ] def get_site_name(self, obj: Capture) -> str: @@ -932,6 +938,6 @@ def get_size(self, obj: Capture) -> int: def get_capture_props(self, obj: Capture) -> dict[str, Any]: return obj.get_opensearch_metadata() or {} - def get_dataset_ids(self, obj: Capture) -> list[str]: - qs = get_capture_datasets(obj, include_deleted=False) + def get_public_dataset_ids(self, obj: Capture) -> list[str]: + qs = obj.datasets.federation_exportable() return [str(dataset.uuid) for dataset in qs] diff --git a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py index 519a4bf79..a8deb619c 100644 --- a/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py +++ b/gateway/sds_gateway/api_methods/serializers/dataset_serializers.py @@ -356,6 +356,8 @@ class Meta(DatasetPublicSerializer.Meta): "capture_count", "capture_file_count", "artifact_file_count", + "is_deleted", + "deleted_at", ] def get_site_name(self, obj: Dataset) -> str: diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_events.py b/gateway/sds_gateway/api_methods/tests/test_federation_events.py index b0e0fd068..777ff3c8c 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_events.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_events.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from unittest.mock import MagicMock from unittest.mock import patch from uuid import uuid4 @@ -9,7 +10,6 @@ import pytest from django.test import override_settings -from sds_gateway.api_methods.federation.events import FederationEventType from sds_gateway.api_methods.federation.events import publish_federation_event from sds_gateway.api_methods.federation.redis_channel import ( resolve_federation_events_channel, @@ -55,14 +55,16 @@ def test_publish_uses_configured_channel( item_uuid = uuid4() publish_federation_event( - event_type=FederationEventType.CREATED, item_type=ItemType.DATASET, uuid=item_uuid, ) mock_client.publish.assert_called_once() - channel, _payload = mock_client.publish.call_args[0] + channel, payload_raw = mock_client.publish.call_args[0] assert channel == "federation:events:crc" + payload = json.loads(payload_raw) + assert payload["item_type"] == ItemType.DATASET.value + assert payload["uuid"] == str(item_uuid) @override_settings( FEDERATION_ENABLED=True, @@ -75,7 +77,6 @@ def test_skips_publish_when_not_operational( mock_get_redis: MagicMock, ) -> None: publish_federation_event( - event_type=FederationEventType.UPDATED, item_type=ItemType.CAPTURE, uuid=uuid4(), ) @@ -89,13 +90,11 @@ class TestFederationSignals: FEDERATION_OPERATIONAL_OVERRIDE=True, FEDERATION_EVENTS_CHANNEL="federation:events:crc", ) - @patch("sds_gateway.api_methods.federation.signals.publish_federation_event") - @patch("sds_gateway.api_methods.federation.signals.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.signals.get_opensearch_client") - @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=False) + @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") + @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") def test_dataset_post_save_indexes_when_published( self, - _mock_exists: MagicMock, _mock_os: MagicMock, mock_indexer_cls: MagicMock, mock_publish: MagicMock, @@ -121,5 +120,8 @@ def test_dataset_post_save_indexes_when_published( assert call["site_name"] == "crc" assert call["item_type"] == ItemType.DATASET assert call["uuid"] == dataset.uuid - assert call["event_type"] == "created" - mock_publish.assert_called_once() + mock_publish.assert_called_once_with( + item_type=ItemType.DATASET, + uuid=dataset.uuid, + timestamp=dataset.updated_at, + ) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py index f34c20484..4179828bf 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py @@ -17,6 +17,9 @@ from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.tests.factories import CaptureFactory from sds_gateway.api_methods.tests.factories import DatasetFactory +from sds_gateway.api_methods.utils.asset_access_control import ( + disconnect_captures_from_dataset, +) pytestmark = pytest.mark.django_db @@ -27,13 +30,11 @@ FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestFederationDatasetSignals(TestCase): - @patch("sds_gateway.api_methods.federation.signals.publish_federation_event") - @patch("sds_gateway.api_methods.federation.signals.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.signals.get_opensearch_client") - @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=False) + @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") + @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") def test_published_dataset_upserts_and_syncs_captures( self, - _mock_exists: MagicMock, _mock_os: MagicMock, mock_indexer_cls: MagicMock, mock_publish: MagicMock, @@ -57,21 +58,21 @@ def test_published_dataset_upserts_and_syncs_captures( assert item_types == {ItemType.DATASET, ItemType.CAPTURE} assert mock_publish.call_count == 2 - @patch("sds_gateway.api_methods.federation.signals.publish_federation_event") - @patch("sds_gateway.api_methods.federation.signals.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.signals.get_opensearch_client") + @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") + @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") @patch( - "sds_gateway.api_methods.federation.signals.fed_doc_exists", - return_value=True, + "sds_gateway.api_methods.federation.reindex.get_federated_export_doc_by_uuid", + return_value={"is_deleted": False}, ) @patch( - "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", - return_value=False, + "sds_gateway.api_methods.federation.reindex.fed_doc_exists", + return_value=True, ) - def test_deleted_dataset_tombstones_when_doc_exists( + def test_deleted_dataset_reindexes_full_body( self, - _mock_tombstone: MagicMock, _mock_exists: MagicMock, + _mock_doc: MagicMock, _mock_os: MagicMock, mock_indexer_cls: MagicMock, mock_publish: MagicMock, @@ -89,88 +90,47 @@ def test_deleted_dataset_tombstones_when_doc_exists( created=False, ) - mock_indexer.apply_local_event.assert_called() - deleted_calls = [ - c - for c in mock_indexer.apply_local_event.call_args_list - if c.kwargs.get("event_type") == "deleted" - ] - assert deleted_calls + mock_indexer.apply_local_event.assert_called_once() + call = mock_indexer.apply_local_event.call_args.kwargs + assert call["item_type"] == ItemType.DATASET + assert call["body"]["is_deleted"] is True + mock_publish.assert_called_once_with( + item_type=ItemType.DATASET, + uuid=dataset.uuid, + timestamp=dataset.updated_at, + ) - @patch("sds_gateway.api_methods.federation.signals._upsert_federated") - @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=False) + @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") def test_draft_dataset_skips_federation( self, - _mock_exists: MagicMock, - mock_upsert: MagicMock, + mock_reindex: MagicMock, ) -> None: dataset = DatasetFactory(status=DatasetStatus.DRAFT, is_public=False) federation_dataset_changed(sender=Dataset, instance=dataset, created=True) - mock_upsert.assert_not_called() + mock_reindex.assert_not_called() - @patch("sds_gateway.api_methods.federation.signals._tombstone_federated_if_exists") - @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=True) + @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") @patch( - "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", - return_value=False, + "sds_gateway.api_methods.federation.reindex.get_federated_export_doc_by_uuid", + return_value={"is_deleted": True}, ) - def test_deleted_dataset_skips_capture_tombstone_when_on_other_published( - self, - _mock_tombstone_flag: MagicMock, - _mock_exists: MagicMock, - mock_tombstone: MagicMock, - ) -> None: - dataset_a = DatasetFactory( - status=DatasetStatus.FINAL, - is_public=True, - is_deleted=True, - ) - dataset_b = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) - capture = CaptureFactory(is_deleted=False) - capture.datasets.add(dataset_a, dataset_b) - - federation_dataset_changed( - sender=Dataset, - instance=dataset_a, - created=False, - ) - - tombstone_calls = [ - c for c in mock_tombstone.call_args_list if c.args[1] == ItemType.DATASET - ] - assert len(tombstone_calls) == 1 - capture_tombstones = [ - c for c in mock_tombstone.call_args_list if c.args[1] == ItemType.CAPTURE - ] - assert not capture_tombstones - - @patch("sds_gateway.api_methods.federation.signals._tombstone_federated_if_exists") - @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=True) @patch( - "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", - return_value=False, + "sds_gateway.api_methods.federation.reindex.fed_doc_exists", + return_value=True, ) - def test_deleted_dataset_tombstones_orphan_capture( + def test_deleted_dataset_skips_when_fed_doc_already_deleted( self, - _mock_tombstone_flag: MagicMock, _mock_exists: MagicMock, - mock_tombstone: MagicMock, + _mock_doc: MagicMock, + mock_reindex: MagicMock, ) -> None: dataset = DatasetFactory( status=DatasetStatus.FINAL, is_public=True, is_deleted=True, ) - capture = CaptureFactory(is_deleted=False) - capture.datasets.add(dataset) - - federation_dataset_changed( - sender=Dataset, - instance=dataset, - created=False, - ) - - assert mock_tombstone.call_count == 2 + federation_dataset_changed(sender=Dataset, instance=dataset, created=False) + mock_reindex.assert_not_called() @override_settings( @@ -179,13 +139,11 @@ def test_deleted_dataset_tombstones_orphan_capture( FEDERATION_OPERATIONAL_OVERRIDE=True, ) class TestFederationCaptureSignals(TestCase): - @patch("sds_gateway.api_methods.federation.signals.publish_federation_event") - @patch("sds_gateway.api_methods.federation.signals.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.signals.get_opensearch_client") - @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=False) + @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") + @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") def test_capture_on_published_dataset_upserts( self, - _mock_exists: MagicMock, _mock_os: MagicMock, mock_indexer_cls: MagicMock, mock_publish: MagicMock, @@ -202,47 +160,103 @@ def test_capture_on_published_dataset_upserts( mock_indexer.apply_local_event.call_args.kwargs["item_type"] == ItemType.CAPTURE ) - mock_publish.assert_called_once() + mock_publish.assert_called_once_with( + item_type=ItemType.CAPTURE, + uuid=capture.uuid, + timestamp=capture.updated_at, + ) - @patch("sds_gateway.api_methods.federation.signals._upsert_federated") - def test_capture_on_draft_dataset_skips_upsert( + @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") + @patch( + "sds_gateway.api_methods.federation.reindex.fed_doc_exists", + return_value=False, + ) + def test_capture_on_draft_only_skips_without_fed_doc( self, - mock_upsert: MagicMock, + _mock_exists: MagicMock, + mock_reindex: MagicMock, ) -> None: dataset = DatasetFactory(status=DatasetStatus.DRAFT, is_public=False) capture = CaptureFactory(is_deleted=False) capture.datasets.add(dataset) federation_capture_changed(sender=Capture, instance=capture) - mock_upsert.assert_not_called() + mock_reindex.assert_not_called() - @patch("sds_gateway.api_methods.federation.signals._tombstone_federated_if_exists") - @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=True) + @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") + @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") @patch( - "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", - return_value=False, + "sds_gateway.api_methods.federation.reindex.get_federated_export_doc_by_uuid", + return_value={"is_deleted": False}, ) - def test_deleted_capture_tombstones_when_doc_exists( + @patch( + "sds_gateway.api_methods.federation.reindex.fed_doc_exists", + return_value=True, + ) + def test_deleted_capture_reindexes_with_is_deleted( self, - _mock_tombstone_flag: MagicMock, _mock_exists: MagicMock, - mock_tombstone: MagicMock, + _mock_doc: MagicMock, + _mock_os: MagicMock, + mock_indexer_cls: MagicMock, + _mock_publish: MagicMock, ) -> None: + mock_indexer = mock_indexer_cls.return_value capture = CaptureFactory(is_deleted=True) federation_capture_changed(sender=Capture, instance=capture) - mock_tombstone.assert_called_once_with(capture, ItemType.CAPTURE) + mock_indexer.apply_local_event.assert_called_once() + assert mock_indexer.apply_local_event.call_args.kwargs["body"]["is_deleted"] - @patch("sds_gateway.api_methods.federation.signals._tombstone_federated_if_exists") - @patch("sds_gateway.api_methods.federation.signals.fed_doc_exists", return_value=True) + +@override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", + FEDERATION_OPERATIONAL_OVERRIDE=True, +) +class TestDatasetDisconnectReindex(TestCase): + @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") + @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") + @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") @patch( - "sds_gateway.api_methods.federation.signals.fed_doc_is_tombstoned", + "sds_gateway.api_methods.federation.reindex.fed_doc_exists", return_value=True, ) - def test_stale_deleted_capture_ignored( + def test_disconnect_captures_reindexes_orphans( self, - _mock_tombstone_flag: MagicMock, _mock_exists: MagicMock, - mock_tombstone: MagicMock, + _mock_os: MagicMock, + mock_indexer_cls: MagicMock, + _mock_publish: MagicMock, ) -> None: - capture = CaptureFactory(is_deleted=True) - federation_capture_changed(sender=Capture, instance=capture) - mock_tombstone.assert_not_called() + mock_indexer = mock_indexer_cls.return_value + dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) + capture = CaptureFactory(is_deleted=False) + capture.datasets.add(dataset) + + disconnect_captures_from_dataset(dataset) + + capture.refresh_from_db() + assert not capture.datasets.exists() + mock_indexer.apply_local_event.assert_called_once() + body = mock_indexer.apply_local_event.call_args.kwargs["body"] + assert body["public_dataset_ids"] == [] + + @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") + def test_dataset_soft_delete_reindexes_orphan_capture( + self, + mock_reindex: MagicMock, + ) -> None: + dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) + capture = CaptureFactory(is_deleted=False) + capture.datasets.add(dataset) + + dataset.soft_delete() + + capture.refresh_from_db() + assert capture.datasets.federation_exportable().exists() is False + capture_calls = [ + c + for c in mock_reindex.call_args_list + if c.args[1] == ItemType.CAPTURE and c.args[0].pk == capture.pk + ] + assert capture_calls diff --git a/gateway/sds_gateway/api_methods/utils/asset_access_control.py b/gateway/sds_gateway/api_methods/utils/asset_access_control.py index 3e1795f1d..56ce8e423 100644 --- a/gateway/sds_gateway/api_methods/utils/asset_access_control.py +++ b/gateway/sds_gateway/api_methods/utils/asset_access_control.py @@ -388,7 +388,18 @@ def disconnect_captures_from_dataset(dataset: Dataset) -> None: Clears ``Capture.datasets`` (M2M) and deprecated ``Capture.dataset`` (FK). """ + from sds_gateway.api_methods.federation.reindex import ( # noqa: PLC0415 + reindex_captures_after_dataset_unlink, + ) + + capture_pks = list( + Capture.datasets.through.objects.filter(dataset_id=dataset.pk).values_list( + "capture_id", + flat=True, + ), + ) Capture.datasets.through.objects.filter(dataset_id=dataset.pk).delete() + reindex_captures_after_dataset_unlink(capture_pks) # TODO: remove FK after contraction Capture.objects.filter(dataset=dataset).update(dataset=None) From 9f04b458025383343c5e16f1c728e5c7c2d81ebe Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 10 Jul 2026 11:43:36 -0400 Subject: [PATCH 14/17] fix: ensure detach assets from datasets excludes published --- .../api_methods/tests/test_file_endpoints.py | 24 ++++++++++++++-- .../api_methods/utils/relationship_utils.py | 28 ++++++++++++++++--- .../api_methods/views/capture_endpoints.py | 15 ++++++---- .../api_methods/views/file_endpoints.py | 13 +++++---- 4 files changed, 64 insertions(+), 16 deletions(-) diff --git a/gateway/sds_gateway/api_methods/tests/test_file_endpoints.py b/gateway/sds_gateway/api_methods/tests/test_file_endpoints.py index 7ee5add9f..ea5c43cea 100644 --- a/gateway/sds_gateway/api_methods/tests/test_file_endpoints.py +++ b/gateway/sds_gateway/api_methods/tests/test_file_endpoints.py @@ -21,6 +21,7 @@ from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import CaptureType +from sds_gateway.api_methods.models import DatasetStatus from sds_gateway.api_methods.models import File from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.serializers.file_serializers import FilePostSerializer @@ -680,7 +681,7 @@ def test_disabled_share_permission_blocks_file_access(self) -> None: ) def test_detach_file_from_datasets(self) -> None: - """PUT detach-from-datasets clears FK and M2M dataset links.""" + """PUT detach-from-datasets clears draft/private FK and M2M dataset links.""" dataset = DatasetFactory(owner=self.user) self.file.datasets.add(dataset) self.file.dataset = dataset @@ -691,8 +692,27 @@ def test_detach_file_from_datasets(self) -> None: assert response.status_code == status.HTTP_200_OK assert response.json()["message"] == ( - "File detached from all connected datasets successfully" + "File detached from draft/private datasets successfully" ) self.file.refresh_from_db() assert self.file.dataset is None assert not self.file.datasets.exists() + + def test_detach_file_keeps_published_dataset_links(self) -> None: + draft = DatasetFactory(owner=self.user, status=DatasetStatus.DRAFT, is_public=False) + published = DatasetFactory( + owner=self.user, + status=DatasetStatus.FINAL, + is_public=True, + ) + self.file.datasets.add(draft, published) + self.file.dataset = draft + self.file.save() + + url = reverse("api:files-detach-from-datasets", args=[str(self.file.uuid)]) + response = self.client.put(url) + + assert response.status_code == status.HTTP_200_OK + self.file.refresh_from_db() + assert list(self.file.datasets.values_list("uuid", flat=True)) == [published.uuid] + assert self.file.dataset is None diff --git a/gateway/sds_gateway/api_methods/utils/relationship_utils.py b/gateway/sds_gateway/api_methods/utils/relationship_utils.py index 02dc5c727..6350fc7e9 100644 --- a/gateway/sds_gateway/api_methods/utils/relationship_utils.py +++ b/gateway/sds_gateway/api_methods/utils/relationship_utils.py @@ -9,10 +9,12 @@ """ from django.db import transaction +from django.db.models import Q from django.db.models import QuerySet from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import DatasetStatus from sds_gateway.api_methods.models import File @@ -305,12 +307,30 @@ def get_capture_datasets( return union_to_queryset(datasets_union, Dataset) +def _datasets_removable_from_item(item: Capture | File) -> QuerySet[Dataset]: + if isinstance(item, Capture): + linked = get_capture_datasets(item, include_deleted=False) + else: + linked = get_file_datasets(item, include_deleted=False) + return linked.exclude( + Q(status=DatasetStatus.FINAL) | + Q(is_public=True) + ) + + @transaction.atomic def detach_item_from_all_datasets(item: Capture | File) -> None: - """Clear deprecated dataset FK and M2M links for a single capture or file.""" - item.dataset = None - item.datasets.clear() - item.save() + """Remove draft/private dataset links; keep published (FINAL or public) links.""" + to_remove = _datasets_removable_from_item(item) + remove_pks = list(to_remove.values_list("pk", flat=True)) + if remove_pks: + item.datasets.remove(*remove_pks) + + fk_dataset = item.dataset + if fk_dataset is not None and not fk_dataset.is_deleted: + if fk_dataset.status != DatasetStatus.FINAL and not fk_dataset.is_public: + item.dataset = None + item.save(update_fields=["dataset"]) def group_captures_by_top_level_dir( diff --git a/gateway/sds_gateway/api_methods/views/capture_endpoints.py b/gateway/sds_gateway/api_methods/views/capture_endpoints.py index 9e2ef28a6..781a69dab 100644 --- a/gateway/sds_gateway/api_methods/views/capture_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/capture_endpoints.py @@ -1077,16 +1077,19 @@ def revoke_share_permissions( ], responses={ 200: OpenApiResponse( - description="Capture detached from all datasets successfully" + description="Capture detached from draft/private datasets successfully" ), 400: OpenApiResponse(description="Bad Request"), 500: OpenApiResponse(description="Internal Server Error"), }, - description="Detach a capture from all connected datasets.", - summary="Detach Capture from All Connected Datasets", + description=( + "Detach a capture from draft and private datasets. " + "Published datasets (final or public) stay linked." + ), + summary="Detach Capture from Draft/Private Datasets", ) def detach_from_datasets(self, request: Request, pk: str | None = None) -> Response: - """Detach a capture from all datasets.""" + """Detach a capture from draft and private datasets (not published).""" if pk is None: return Response( {"detail": "Capture UUID is required."}, @@ -1100,7 +1103,9 @@ def detach_from_datasets(self, request: Request, pk: str | None = None) -> Respo return Response( status=status.HTTP_200_OK, data={ - "message": "Capture detached from all connected datasets successfully", + "message": ( + "Capture detached from draft/private datasets successfully" + ), }, ) diff --git a/gateway/sds_gateway/api_methods/views/file_endpoints.py b/gateway/sds_gateway/api_methods/views/file_endpoints.py index 1e9308df8..6b5444afc 100644 --- a/gateway/sds_gateway/api_methods/views/file_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/file_endpoints.py @@ -559,16 +559,19 @@ def update(self, request: Request, pk: str | None = None) -> Response: ], responses={ 200: OpenApiResponse( - description="File detached from all datasets successfully" + description="File detached from draft/private datasets successfully" ), 400: OpenApiResponse(description="Bad Request"), 500: OpenApiResponse(description="Internal Server Error"), }, - description="Detach a file from all datasets.", - summary="Detach File from All Connected Datasets", + description=( + "Detach a file from draft and private datasets. " + "Published datasets (final or public) stay linked." + ), + summary="Detach File from Draft/Private Datasets", ) def detach_from_datasets(self, request: Request, pk: str | None = None) -> Response: - """Detach a file from all datasets.""" + """Detach a file from draft and private datasets (not published).""" if pk is None: return Response( {"detail": "File UUID is required."}, @@ -582,7 +585,7 @@ def detach_from_datasets(self, request: Request, pk: str | None = None) -> Respo return Response( status=status.HTTP_200_OK, data={ - "message": "File detached from all connected datasets successfully", + "message": "File detached from draft/private datasets successfully", }, ) From 4609c03bcf23a7264ac05d23d8d7c6572be7bcbc Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 10 Jul 2026 12:29:52 -0400 Subject: [PATCH 15/17] fix: fail state exceptions --- .../federation/compile_federated_data.py | 10 +++ .../api_methods/federation/reindex.py | 58 ++++++++++++++- .../api_methods/federation/signals.py | 24 +----- .../tests/test_federation_events.py | 19 +++-- .../tests/test_federation_hardening.py | 29 ++++++++ .../tests/test_federation_signals.py | 74 ++++++++++++++----- 6 files changed, 169 insertions(+), 45 deletions(-) diff --git a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py index 4d47d5c28..ae7407139 100644 --- a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py +++ b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py @@ -7,6 +7,8 @@ from uuid import UUID from django.conf import settings +from loguru import logger as log +from opensearchpy import exceptions as os_exceptions from opensearchpy.exceptions import NotFoundError from sds_gateway.api_methods.federation.fed_index import index_for_item_type @@ -90,6 +92,14 @@ def get_federated_export_doc_by_uuid( response = client.get(index=index, id=doc_id) except NotFoundError: return None + except os_exceptions.OpenSearchException as exc: + log.warning( + "Federation fed doc lookup failed for {} {}: {}", + asset_type.value, + asset_uuid, + exc, + ) + return None source = response.get("_source") if not isinstance(source, dict): return None diff --git a/gateway/sds_gateway/api_methods/federation/reindex.py b/gateway/sds_gateway/api_methods/federation/reindex.py index 204ef3324..60249bc82 100644 --- a/gateway/sds_gateway/api_methods/federation/reindex.py +++ b/gateway/sds_gateway/api_methods/federation/reindex.py @@ -4,6 +4,7 @@ from collections.abc import Iterable +from django.db import transaction from loguru import logger as log from sds_gateway.api_methods.federation.availability import is_federation_operational @@ -55,13 +56,64 @@ def reindex_federated_asset( ) +def schedule_federation_dataset_reindex(dataset: Dataset) -> None: + """Run dataset (+ exportable member captures) reindex after transaction commit.""" + dataset_pk = dataset.pk + + def _on_commit() -> None: + try: + fresh = Dataset.objects.get(pk=dataset_pk) + except Dataset.DoesNotExist: + return + try: + if not dataset_needs_federation_reindex(fresh): + return + reindex_federated_asset(fresh, ItemType.DATASET) + if fresh.is_deleted or not fresh.is_federation_exportable(): + return + for capture in fresh.captures.filter(is_deleted=False): + if capture_needs_federation_reindex(capture): + reindex_federated_asset(capture, ItemType.CAPTURE) + except Exception as exc: # noqa: BLE001 + log.warning("Federation dataset reindex on commit failed: {}", exc) + + transaction.on_commit(_on_commit, robust=True) + + +def schedule_federation_capture_reindex(capture: Capture) -> None: + """Run capture reindex after transaction commit.""" + capture_pk = capture.pk + + def _on_commit() -> None: + try: + fresh = Capture.objects.get(pk=capture_pk) + except Capture.DoesNotExist: + return + try: + if not capture_needs_federation_reindex(fresh): + return + reindex_federated_asset(fresh, ItemType.CAPTURE) + except Exception as exc: # noqa: BLE001 + log.warning("Federation capture reindex on commit failed: {}", exc) + + transaction.on_commit(_on_commit, robust=True) + + def reindex_captures_after_dataset_unlink(capture_pks: Iterable[int]) -> None: + """Reindex captures unlinked from a dataset after transaction commit.""" pks = list(capture_pks) if not pks: return - for capture in Capture.objects.filter(pk__in=pks): - if capture_needs_federation_reindex(capture): - reindex_federated_asset(capture, ItemType.CAPTURE) + + def _on_commit() -> None: + try: + for capture in Capture.objects.filter(pk__in=pks): + if capture_needs_federation_reindex(capture): + reindex_federated_asset(capture, ItemType.CAPTURE) + except Exception as exc: # noqa: BLE001 + log.warning("Federation capture unlink reindex on commit failed: {}", exc) + + transaction.on_commit(_on_commit, robust=True) def _fed_doc_shows_is_deleted( diff --git a/gateway/sds_gateway/api_methods/federation/signals.py b/gateway/sds_gateway/api_methods/federation/signals.py index 6cdc0a530..a30b09cb7 100644 --- a/gateway/sds_gateway/api_methods/federation/signals.py +++ b/gateway/sds_gateway/api_methods/federation/signals.py @@ -6,18 +6,10 @@ from django.dispatch import receiver from sds_gateway.api_methods.federation.availability import is_federation_operational -from sds_gateway.api_methods.federation.reindex import capture_needs_federation_reindex -from sds_gateway.api_methods.federation.reindex import dataset_needs_federation_reindex -from sds_gateway.api_methods.federation.reindex import reindex_federated_asset +from sds_gateway.api_methods.federation.reindex import schedule_federation_capture_reindex +from sds_gateway.api_methods.federation.reindex import schedule_federation_dataset_reindex from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset -from sds_gateway.api_methods.models import ItemType - - -def _reindex_dataset_captures(dataset: Dataset) -> None: - for capture in dataset.captures.filter(is_deleted=False): - if capture_needs_federation_reindex(capture): - reindex_federated_asset(capture, ItemType.CAPTURE) @receiver(post_save, sender=Dataset) @@ -29,13 +21,7 @@ def federation_dataset_changed( ) -> None: if not is_federation_operational(): return - if not dataset_needs_federation_reindex(instance): - return - - reindex_federated_asset(instance, ItemType.DATASET) - if instance.is_deleted or not instance.is_federation_exportable(): - return - _reindex_dataset_captures(instance) + schedule_federation_dataset_reindex(instance) @receiver(post_save, sender=Capture) @@ -46,6 +32,4 @@ def federation_capture_changed( ) -> None: if not is_federation_operational(): return - if not capture_needs_federation_reindex(instance): - return - reindex_federated_asset(instance, ItemType.CAPTURE) + schedule_federation_capture_reindex(instance) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_events.py b/gateway/sds_gateway/api_methods/tests/test_federation_events.py index 777ff3c8c..14f72943f 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_events.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_events.py @@ -3,11 +3,13 @@ from __future__ import annotations import json +from contextlib import contextmanager from unittest.mock import MagicMock from unittest.mock import patch from uuid import uuid4 import pytest +from django.test import TestCase from django.test import override_settings from sds_gateway.api_methods.federation.events import publish_federation_event @@ -19,6 +21,12 @@ pytestmark = pytest.mark.django_db +@contextmanager +def _federation_on_commit(): + with TestCase.captureOnCommitCallbacks(execute=True): + yield + + class TestResolveFederationEventsChannel: def test_site_prefixed_when_site_name_set(self) -> None: assert ( @@ -109,11 +117,12 @@ def test_dataset_post_save_indexes_when_published( status=DatasetStatus.FINAL, is_public=True, ) - federation_dataset_changed( - sender=Dataset, - instance=dataset, - created=False, - ) + with _federation_on_commit(): + federation_dataset_changed( + sender=Dataset, + instance=dataset, + created=False, + ) mock_indexer.apply_local_event.assert_called_once() call = mock_indexer.apply_local_event.call_args.kwargs diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py index 6df5d3609..a63c4a190 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -256,3 +256,32 @@ def test_export_returns_503_when_not_operational(self) -> None: **self._auth(self.sync_key), ) assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE + + +class TestFederationOpenSearchReads: + @patch("sds_gateway.api_methods.federation.compile_federated_data.get_opensearch_client") + def test_fed_doc_lookup_treats_cluster_errors_as_missing( + self, + mock_get_client: MagicMock, + ) -> None: + from opensearchpy import exceptions as os_exceptions + + from sds_gateway.api_methods.federation.compile_federated_data import ( + fed_doc_exists, + ) + from sds_gateway.api_methods.models import ItemType + from sds_gateway.api_methods.tests.factories import DatasetFactory + + mock_get_client.return_value.get.side_effect = os_exceptions.ConnectionError( + "connection failed", + ) + dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) + + assert ( + fed_doc_exists( + asset_type=ItemType.DATASET, + site_name="crc", + instance=dataset, + ) + is False + ) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py index 4179828bf..ffba6846a 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextlib import contextmanager from unittest.mock import MagicMock from unittest.mock import patch @@ -24,6 +25,12 @@ pytestmark = pytest.mark.django_db +@contextmanager +def _federation_on_commit(): + with TestCase.captureOnCommitCallbacks(execute=True): + yield + + @override_settings( FEDERATION_ENABLED=True, FEDERATION_SITE_NAME="crc", @@ -44,11 +51,12 @@ def test_published_dataset_upserts_and_syncs_captures( capture = CaptureFactory(is_deleted=False) capture.datasets.add(dataset) - federation_dataset_changed( - sender=Dataset, - instance=dataset, - created=False, - ) + with _federation_on_commit(): + federation_dataset_changed( + sender=Dataset, + instance=dataset, + created=False, + ) assert mock_indexer.apply_local_event.call_count == 2 item_types = { @@ -84,11 +92,12 @@ def test_deleted_dataset_reindexes_full_body( is_deleted=True, ) - federation_dataset_changed( - sender=Dataset, - instance=dataset, - created=False, - ) + with _federation_on_commit(): + federation_dataset_changed( + sender=Dataset, + instance=dataset, + created=False, + ) mock_indexer.apply_local_event.assert_called_once() call = mock_indexer.apply_local_event.call_args.kwargs @@ -106,7 +115,8 @@ def test_draft_dataset_skips_federation( mock_reindex: MagicMock, ) -> None: dataset = DatasetFactory(status=DatasetStatus.DRAFT, is_public=False) - federation_dataset_changed(sender=Dataset, instance=dataset, created=True) + with _federation_on_commit(): + federation_dataset_changed(sender=Dataset, instance=dataset, created=True) mock_reindex.assert_not_called() @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") @@ -129,7 +139,8 @@ def test_deleted_dataset_skips_when_fed_doc_already_deleted( is_public=True, is_deleted=True, ) - federation_dataset_changed(sender=Dataset, instance=dataset, created=False) + with _federation_on_commit(): + federation_dataset_changed(sender=Dataset, instance=dataset, created=False) mock_reindex.assert_not_called() @@ -153,7 +164,8 @@ def test_capture_on_published_dataset_upserts( capture = CaptureFactory(is_deleted=False) capture.datasets.add(dataset) - federation_capture_changed(sender=Capture, instance=capture) + with _federation_on_commit(): + federation_capture_changed(sender=Capture, instance=capture) mock_indexer.apply_local_event.assert_called_once() assert ( @@ -179,7 +191,8 @@ def test_capture_on_draft_only_skips_without_fed_doc( dataset = DatasetFactory(status=DatasetStatus.DRAFT, is_public=False) capture = CaptureFactory(is_deleted=False) capture.datasets.add(dataset) - federation_capture_changed(sender=Capture, instance=capture) + with _federation_on_commit(): + federation_capture_changed(sender=Capture, instance=capture) mock_reindex.assert_not_called() @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") @@ -203,7 +216,8 @@ def test_deleted_capture_reindexes_with_is_deleted( ) -> None: mock_indexer = mock_indexer_cls.return_value capture = CaptureFactory(is_deleted=True) - federation_capture_changed(sender=Capture, instance=capture) + with _federation_on_commit(): + federation_capture_changed(sender=Capture, instance=capture) mock_indexer.apply_local_event.assert_called_once() assert mock_indexer.apply_local_event.call_args.kwargs["body"]["is_deleted"] @@ -233,7 +247,8 @@ def test_disconnect_captures_reindexes_orphans( capture = CaptureFactory(is_deleted=False) capture.datasets.add(dataset) - disconnect_captures_from_dataset(dataset) + with _federation_on_commit(): + disconnect_captures_from_dataset(dataset) capture.refresh_from_db() assert not capture.datasets.exists() @@ -250,7 +265,8 @@ def test_dataset_soft_delete_reindexes_orphan_capture( capture = CaptureFactory(is_deleted=False) capture.datasets.add(dataset) - dataset.soft_delete() + with _federation_on_commit(): + dataset.soft_delete() capture.refresh_from_db() assert capture.datasets.federation_exportable().exists() is False @@ -260,3 +276,27 @@ def test_dataset_soft_delete_reindexes_orphan_capture( if c.args[1] == ItemType.CAPTURE and c.args[0].pk == capture.pk ] assert capture_calls + + +@override_settings( + FEDERATION_ENABLED=True, + FEDERATION_SITE_NAME="crc", + FEDERATION_OPERATIONAL_OVERRIDE=True, +) +class TestFederationReindexOnCommit(TestCase): + @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") + def test_rollback_skips_federation_reindex( + self, + mock_reindex: MagicMock, + ) -> None: + from django.db import transaction + + dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) + try: + with transaction.atomic(): + dataset.name = "rolled back" + dataset.save() + raise RuntimeError("abort") + except RuntimeError: + pass + mock_reindex.assert_not_called() From 38c54350c2329a50f5e1b372ca317fe5c83f474e Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 10 Jul 2026 13:54:56 -0400 Subject: [PATCH 16/17] fix tests --- gateway/compose.local.yaml | 2 +- gateway/sds_gateway/api_methods/apps.py | 2 +- .../sds_gateway/api_methods/authentication.py | 3 +-- .../api_methods/tests/factories.py | 15 +++++------ .../api_methods/tests/test_authenticate.py | 5 ++-- .../tests/test_dataset_endpoints.py | 2 ++ .../tests/test_federation_export.py | 6 ++--- .../tests/test_federation_hardening.py | 4 ++- .../tests/test_federation_signals.py | 5 ++++ .../tests/test_federation_sync_api_key.py | 25 +++++++++++-------- gateway/sds_gateway/users/api/views.py | 6 ++--- .../tests/test_federation_sync_drf_init.py | 6 +++-- .../users/tests/test_quick_add_views.py | 5 +++- 13 files changed, 51 insertions(+), 35 deletions(-) diff --git a/gateway/compose.local.yaml b/gateway/compose.local.yaml index 0f0406aa9..c9d4d25aa 100644 --- a/gateway/compose.local.yaml +++ b/gateway/compose.local.yaml @@ -83,7 +83,7 @@ services: - ./.envs/local/storage.env - ./.envs/local/postgres.env - ./.envs/local/opensearch.env - - ../../federation-shared.env + - ../federation-shared.env environment: - SKIP_POSTGRES_WAIT=1 # remember /entrypoint runs first diff --git a/gateway/sds_gateway/api_methods/apps.py b/gateway/sds_gateway/api_methods/apps.py index 023e95e5c..2d6c77820 100644 --- a/gateway/sds_gateway/api_methods/apps.py +++ b/gateway/sds_gateway/api_methods/apps.py @@ -33,7 +33,7 @@ def ready(self) -> None: initialize_federation_operational_state, ) - def _init_federation_after_migrate(_sender, **kwargs) -> None: + def _init_federation_after_migrate(sender, **kwargs) -> None: # noqa: ARG001 initialize_federation_operational_state() post_migrate.connect( diff --git a/gateway/sds_gateway/api_methods/authentication.py b/gateway/sds_gateway/api_methods/authentication.py index 60e831970..7dfa520a9 100644 --- a/gateway/sds_gateway/api_methods/authentication.py +++ b/gateway/sds_gateway/api_methods/authentication.py @@ -25,8 +25,7 @@ def authenticate(self, request) -> tuple[User, UserAPIKey]: """ auth_header = request.headers.get("Authorization") if not auth_header: - msg = "No API key provided" - raise AuthenticationFailed(msg) + return None # extract key from header split_header = auth_header.split(":") diff --git a/gateway/sds_gateway/api_methods/tests/factories.py b/gateway/sds_gateway/api_methods/tests/factories.py index 31cab86d2..39defd8f1 100644 --- a/gateway/sds_gateway/api_methods/tests/factories.py +++ b/gateway/sds_gateway/api_methods/tests/factories.py @@ -17,6 +17,7 @@ from factory import LazyAttribute from factory import LazyFunction from factory import Sequence +from factory import SubFactory from factory import post_generation from factory.django import DjangoModelFactory from faker import Faker as FakerInstance @@ -103,7 +104,7 @@ class DatasetFactory(DjangoModelFactory): provenance = {"source": "test"} citation = {"title": "Test Dataset"} other = {"notes": "Test dataset"} - owner = FactoryFaker("subfactory", factory=UserFactory) + owner = SubFactory(UserFactory) is_deleted = False is_public = False @@ -197,7 +198,7 @@ class FileFactory(DjangoModelFactory): permissions = "rw-r--r--" size = FactoryFaker("random_int", min=1000, max=1000000) sum_blake3 = FactoryFaker("sha256") - owner = FactoryFaker("subfactory", factory=UserFactory) + owner = SubFactory(UserFactory) is_deleted = False @post_generation @@ -239,7 +240,7 @@ class Meta: channel = FactoryFaker("word") capture_type = "drf" top_level_dir = LazyFunction(lambda: _faker.file_path(depth=2).replace("/", "_")) - owner = FactoryFaker("subfactory", factory=UserFactory) + owner = SubFactory(UserFactory) name = FactoryFaker("slug") index_name = "captures-drf" @@ -260,8 +261,8 @@ class DRFDataFileFactory(DjangoModelFactory): permissions = "rw-r----" size = FactoryFaker("random_int", min=1000, max=1000000) sum_blake3 = FactoryFaker("sha256") - owner = FactoryFaker("subfactory", factory=UserFactory) - capture = FactoryFaker("subfactory", factory=CaptureFactory) + owner = SubFactory(UserFactory) + capture = SubFactory(CaptureFactory) is_deleted = False @post_generation @@ -310,8 +311,8 @@ class UserSharePermissionFactory(DjangoModelFactory): permission = UserSharePermissionFactory(is_enabled=False) """ - owner = FactoryFaker("subfactory", factory=UserFactory) - shared_with = FactoryFaker("subfactory", factory=UserFactory) + owner = SubFactory(UserFactory) + shared_with = SubFactory(UserFactory) item_type = FactoryFaker( "random_element", elements=[ItemType.DATASET, ItemType.CAPTURE] ) diff --git a/gateway/sds_gateway/api_methods/tests/test_authenticate.py b/gateway/sds_gateway/api_methods/tests/test_authenticate.py index 7c9620213..7a0737720 100644 --- a/gateway/sds_gateway/api_methods/tests/test_authenticate.py +++ b/gateway/sds_gateway/api_methods/tests/test_authenticate.py @@ -39,10 +39,9 @@ def test_auth_valid_key(self) -> None: assert user.email == "test@example.com" def test_auth_no_key(self) -> None: - """Makes sure that a request without an API key raises.""" + """Without a key, authentication is deferred to other classes.""" request = self.factory.get("/auth/") - with self.assertRaises(AuthenticationFailed): # noqa: PT027 - self.auth.authenticate(request) + assert self.auth.authenticate(request) is None def test_auth_invalid_key(self) -> None: """Makes sure that a request with an invalid API key raises.""" diff --git a/gateway/sds_gateway/api_methods/tests/test_dataset_endpoints.py b/gateway/sds_gateway/api_methods/tests/test_dataset_endpoints.py index f618ff123..6e247078a 100644 --- a/gateway/sds_gateway/api_methods/tests/test_dataset_endpoints.py +++ b/gateway/sds_gateway/api_methods/tests/test_dataset_endpoints.py @@ -638,6 +638,8 @@ def test_get_dataset_files_via_api_key(self): def test_get_dataset_files_via_session(self): """Dataset files manifest accepts session authentication.""" + self.user.is_approved = True + self.user.save(update_fields=["is_approved"]) self._attach_artifact_to_dataset() client = APIClient() client.force_login(self.user) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export.py b/gateway/sds_gateway/api_methods/tests/test_federation_export.py index 34329528a..bcabcc96a 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export.py @@ -55,10 +55,8 @@ def setUp(self) -> None: status=DatasetStatus.DRAFT, keywords=None, ) - self.public_capture = CaptureFactory( - owner=self.owner, - datasets=[self.public_dataset], - ) + self.public_capture = CaptureFactory(owner=self.owner) + self.public_capture.datasets.add(self.public_dataset) self.list_datasets_url = reverse( "api:federation-export-datasets-list", ) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py index a63c4a190..695fd64a2 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -272,8 +272,10 @@ def test_fed_doc_lookup_treats_cluster_errors_as_missing( from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.tests.factories import DatasetFactory - mock_get_client.return_value.get.side_effect = os_exceptions.ConnectionError( + mock_get_client.return_value.get.side_effect = os_exceptions.OpenSearchException( + 503, "connection failed", + {}, ) dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py index ffba6846a..325e5d674 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py @@ -257,8 +257,13 @@ def test_disconnect_captures_reindexes_orphans( assert body["public_dataset_ids"] == [] @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") + @patch( + "sds_gateway.api_methods.federation.reindex.fed_doc_exists", + return_value=True, + ) def test_dataset_soft_delete_reindexes_orphan_capture( self, + _mock_exists: MagicMock, mock_reindex: MagicMock, ) -> None: dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py b/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py index 395968fc1..47359baed 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_sync_api_key.py @@ -3,6 +3,7 @@ from __future__ import annotations import pytest +from django.conf import settings from django.urls import reverse from rest_framework import status from rest_framework.authtoken.models import Token @@ -19,14 +20,15 @@ class TestFederationSyncMint: def setup_method(self) -> None: - User.objects.filter(email="federation-sync@internal.local").delete() + email = settings.FEDERATION_SYNC_USER_EMAIL + User.objects.filter(email=email).delete() update_service_server_token( - user_email="federation-sync@internal.local", + user_email=email, token_key="a" * 40, ) def test_mint_creates_api_key(self) -> None: - user = User.objects.get(email="federation-sync@internal.local") + user = User.objects.get(email=settings.FEDERATION_SYNC_USER_EMAIL) raw = mint_federation_sync_api_key(user) assert raw is not None key = UserAPIKey.objects.get_from_key(raw) @@ -34,7 +36,7 @@ def test_mint_creates_api_key(self) -> None: assert key.user_id == user.pk def test_mint_replaces_prior_key_like_svi(self) -> None: - user = User.objects.get(email="federation-sync@internal.local") + user = User.objects.get(email=settings.FEDERATION_SYNC_USER_EMAIL) first = mint_federation_sync_api_key(user) second = mint_federation_sync_api_key(user) assert second != first @@ -49,21 +51,24 @@ def test_mint_replaces_prior_key_like_svi(self) -> None: class TestGetFederationSyncApiKeyEndpoint: def setup_method(self) -> None: - User.objects.filter(email="federation-sync@internal.local").delete() + email = settings.FEDERATION_SYNC_USER_EMAIL + User.objects.filter(email=email).delete() update_service_server_token( - user_email="federation-sync@internal.local", + user_email=email, token_key="b" * 40, ) def test_mint_via_http(self) -> None: - user = User.objects.get(email="federation-sync@internal.local") - token = Token.objects.get(user=user) + sync_email = settings.FEDERATION_SYNC_USER_EMAIL + sync_user = User.objects.get(email=sync_email) + target = User.objects.create(email="mint-target@example.com", is_approved=True) + token = Token.objects.get(user=sync_user) client = APIClient() url = reverse("users:get_federation_sync_api_key") response = client.get( - url, - {"email": user.email}, + f"{url}?email={target.email}", HTTP_AUTHORIZATION=f"Token {token.key}", ) assert response.status_code == status.HTTP_200_OK assert "api_key" in response.json() + assert response.json()["email"] == target.email diff --git a/gateway/sds_gateway/users/api/views.py b/gateway/sds_gateway/users/api/views.py index dec491b8b..c4b288b09 100644 --- a/gateway/sds_gateway/users/api/views.py +++ b/gateway/sds_gateway/users/api/views.py @@ -55,9 +55,9 @@ class BackendServiceMintAPIKeyView(APIView): permission_classes = [IsAuthenticated] authentication_classes = [TokenAuthentication] - service_user_email_setting: str - unauthorized_message: str - mint_user_api_key: Callable[[User], str] + service_user_email_setting = "" + unauthorized_message = "" + mint_user_api_key: Callable[[User], str] | None = None def get(self, request: Request) -> Response: allowed_email = getattr(settings, self.service_user_email_setting) diff --git a/gateway/sds_gateway/users/tests/test_federation_sync_drf_init.py b/gateway/sds_gateway/users/tests/test_federation_sync_drf_init.py index 4fa39578c..bac764ccb 100644 --- a/gateway/sds_gateway/users/tests/test_federation_sync_drf_init.py +++ b/gateway/sds_gateway/users/tests/test_federation_sync_drf_init.py @@ -3,6 +3,7 @@ from __future__ import annotations import pytest +from django.conf import settings from django.test import override_settings from rest_framework.authtoken.models import Token @@ -14,11 +15,12 @@ @override_settings(FEDERATION_SYNC_DRF_TOKEN="a" * 40) def test_update_federation_sync_drf_token_from_settings() -> None: - User.objects.filter(email="federation-sync@internal.local").delete() + email = settings.FEDERATION_SYNC_USER_EMAIL + User.objects.filter(email=email).delete() update_federation_sync_drf_token() - user = User.objects.get(email="federation-sync@internal.local") + user = User.objects.get(email=email) assert Token.objects.get(user=user).key == "a" * 40 diff --git a/gateway/sds_gateway/users/tests/test_quick_add_views.py b/gateway/sds_gateway/users/tests/test_quick_add_views.py index 5526a0918..76bd8712b 100644 --- a/gateway/sds_gateway/users/tests/test_quick_add_views.py +++ b/gateway/sds_gateway/users/tests/test_quick_add_views.py @@ -11,6 +11,7 @@ from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import CaptureType +from sds_gateway.api_methods.models import Dataset from sds_gateway.api_methods.models import DatasetStatus from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.models import PermissionLevel @@ -509,7 +510,9 @@ def test_response_contains_uuid_and_name(self, client: Client, owner: User): assert entry["name"] == "My Dataset" def test_unnamed_dataset_shows_fallback(self, client: Client, owner: User): - ds = DatasetFactory(owner=owner, is_public=False, name="", keywords=None) + ds = DatasetFactory(owner=owner, is_public=False, keywords=None) + Dataset.objects.filter(pk=ds.pk).update(name="") + ds.refresh_from_db() client.force_login(owner) response = self._get(client) From 94ef4d5a171317cb7959a5ea40052ba3f2d23f99 Mon Sep 17 00:00:00 2001 From: klpoland Date: Fri, 10 Jul 2026 14:01:02 -0400 Subject: [PATCH 17/17] linting, hooks --- .gitignore | 2 +- gateway/config/settings/base.py | 18 ++-- gateway/sds_gateway/api_methods/apps.py | 2 +- .../api_methods/federation/availability.py | 12 +-- .../federation/compile_federated_data.py | 18 ++-- .../api_methods/federation/fed_index.py | 10 +- .../api_methods/federation/reindex.py | 21 ++++- .../api_methods/federation/signals.py | 10 +- .../helpers/extract_drf_metadata.py | 2 +- gateway/sds_gateway/api_methods/models.py | 91 ++++++++++--------- .../tests/test_federation_events.py | 15 +-- .../tests/test_federation_export.py | 4 +- .../tests/test_federation_export_contract.py | 2 +- .../tests/test_federation_hardening.py | 38 ++++---- .../tests/test_federation_signals.py | 74 +++++++-------- .../api_methods/tests/test_file_endpoints.py | 8 +- .../api_methods/utils/relationship_utils.py | 5 +- .../api_methods/views/federation_endpoints.py | 8 +- .../users/backend_service_key_utils.py | 4 +- .../sds_gateway/users/tests/test_svi_utils.py | 2 +- 20 files changed, 186 insertions(+), 160 deletions(-) diff --git a/.gitignore b/.gitignore index 4c5655055..ae347f3cd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ **/agents.md agents.md htmlcov/ -federation-shared.env \ No newline at end of file +federation-shared.env diff --git a/gateway/config/settings/base.py b/gateway/config/settings/base.py index 9914549e0..92efe5e4d 100644 --- a/gateway/config/settings/base.py +++ b/gateway/config/settings/base.py @@ -10,13 +10,13 @@ from celery.schedules import crontab from environs import env +from sds_gateway.api_methods.federation.redis_channel import ( + resolve_federation_events_channel, +) from config.settings.logs import ColoredFormatter from config.settings.utils import guess_admin_console_env from config.settings.utils import guess_max_web_download_size -from sds_gateway.api_methods.federation.redis_channel import ( - resolve_federation_events_channel, -) __rng = random.SystemRandom() @@ -751,7 +751,8 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: SDS_PROGRAMMATIC_SITE_NAME: str = env.str("SDS_PROGRAMMATIC_SITE_NAME", default="sds") SDS_SITE_FQDN: str = env.str("SDS_SITE_FQDN", default="localhost") -# Federation peer short name (RFC [site].name, e.g. crc, haystack); not SDS_PROGRAMMATIC_SITE_NAME. +# Federation peer short name (RFC [site].name, e.g. crc, haystack); +# not SDS_PROGRAMMATIC_SITE_NAME. FEDERATION_SITE_NAME: str = env.str("FEDERATION_SITE_NAME", default="").strip() # Master switch: when False, federation export and Redis events are inactive. FEDERATION_ENABLED: bool = env.bool("FEDERATION_ENABLED", default=False) @@ -767,7 +768,8 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: "FEDERATION_SYNC_USER_EMAIL", default="federation-sync@internal.local", ) -# From federation-shared.env (compose env_file); used by init_federation_sync_token only. +# From federation-shared.env (compose env_file); +# used by init_federation_sync_token only. FEDERATION_SYNC_DRF_TOKEN: str = env.str("FEDERATION_SYNC_DRF_TOKEN", default="") FEDERATION_SYNC_HEALTH_URL: str = env.str( "FEDERATION_SYNC_HEALTH_URL", @@ -795,9 +797,9 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str: # Tests may set via override_settings without running probes. FEDERATION_OPERATIONAL_OVERRIDE: bool | None = None # Export API: internal Docker/private networks (sync → django on sds-network). -FEDERATION_EXPORT_ALLOWED_CIDRS: list[ - ipaddress.IPv4Network | ipaddress.IPv6Network -] = _federation_export_allowed_cidrs_from_env() +FEDERATION_EXPORT_ALLOWED_CIDRS: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = ( + _federation_export_allowed_cidrs_from_env() +) # ADMIN_CONSOLE_ENV is used to visually distinguish between different environments # (production, staging, local) in the admin console and error emails. It does not affect diff --git a/gateway/sds_gateway/api_methods/apps.py b/gateway/sds_gateway/api_methods/apps.py index 2d6c77820..a9b6150af 100644 --- a/gateway/sds_gateway/api_methods/apps.py +++ b/gateway/sds_gateway/api_methods/apps.py @@ -33,7 +33,7 @@ def ready(self) -> None: initialize_federation_operational_state, ) - def _init_federation_after_migrate(sender, **kwargs) -> None: # noqa: ARG001 + def _init_federation_after_migrate(sender, **kwargs) -> None: initialize_federation_operational_state() post_migrate.connect( diff --git a/gateway/sds_gateway/api_methods/federation/availability.py b/gateway/sds_gateway/api_methods/federation/availability.py index eaaf32e59..dcca8aada 100644 --- a/gateway/sds_gateway/api_methods/federation/availability.py +++ b/gateway/sds_gateway/api_methods/federation/availability.py @@ -9,7 +9,10 @@ import urllib.request from typing import Any +from config.settings.base import FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT +from config.settings.base import _parse_cidrs from django.conf import settings +from django.db import connection from django.db.utils import DatabaseError from loguru import logger as log @@ -41,9 +44,6 @@ def _export_allowed_networks() -> list[ipaddress.IPv4Network | ipaddress.IPv6Net continue networks.append(ipaddress.ip_network(text, strict=False)) if not networks: - from config.settings.base import FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT - from config.settings.base import _parse_cidrs - networks = _parse_cidrs(FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT) return networks @@ -70,7 +70,7 @@ def is_client_ip_allowed_for_federation_export(request) -> bool: return any(addr in network for network in cidrs) -def _sync_health_ok() -> tuple[bool, str]: # noqa: PLR0911 +def _sync_health_ok() -> tuple[bool, str]: # noqa: C901, PLR0911 if _setting("FEDERATION_SKIP_SYNC_HEALTH_PROBE", default=False): return True, "health probe skipped" url = (_setting("FEDERATION_SYNC_HEALTH_URL") or "").strip() @@ -173,10 +173,8 @@ def federation_operational_db_ready() -> bool: """False when federation probes would hit tables that are not migrated yet.""" if not _setting("FEDERATION_ENABLED", default=False): return True - table = UserAPIKey._meta.db_table + table = UserAPIKey._meta.db_table # noqa: SLF001 try: - from django.db import connection - return table in connection.introspection.table_names() except DatabaseError: return False diff --git a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py index ae7407139..29be27ffb 100644 --- a/gateway/sds_gateway/api_methods/federation/compile_federated_data.py +++ b/gateway/sds_gateway/api_methods/federation/compile_federated_data.py @@ -4,15 +4,14 @@ from typing import TYPE_CHECKING from typing import Any -from uuid import UUID from django.conf import settings from loguru import logger as log from opensearchpy import exceptions as os_exceptions from opensearchpy.exceptions import NotFoundError -from sds_gateway.api_methods.federation.fed_index import index_for_item_type from sds_gateway.api_methods.federation.fed_index import federated_doc_id +from sds_gateway.api_methods.federation.fed_index import index_for_item_type from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset from sds_gateway.api_methods.models import ItemType @@ -25,6 +24,8 @@ from sds_gateway.api_methods.utils.opensearch_client import get_opensearch_client if TYPE_CHECKING: + from uuid import UUID + from django.db.models import QuerySet @@ -112,11 +113,14 @@ def fed_doc_exists( site_name: str, instance: Dataset | Capture, ) -> bool: - return get_federated_export_doc_by_uuid( - asset_type=asset_type, - asset_uuid=instance.uuid, - site_name=site_name, - ) is not None + return ( + get_federated_export_doc_by_uuid( + asset_type=asset_type, + asset_uuid=instance.uuid, + site_name=site_name, + ) + is not None + ) def public_datasets_queryset() -> QuerySet[Dataset]: diff --git a/gateway/sds_gateway/api_methods/federation/fed_index.py b/gateway/sds_gateway/api_methods/federation/fed_index.py index e255d6773..3a338a4da 100644 --- a/gateway/sds_gateway/api_methods/federation/fed_index.py +++ b/gateway/sds_gateway/api_methods/federation/fed_index.py @@ -2,13 +2,17 @@ from __future__ import annotations -from datetime import datetime +from typing import TYPE_CHECKING from typing import Any -from uuid import UUID -from opensearchpy import OpenSearch from sds_gateway.api_methods.models import ItemType +if TYPE_CHECKING: + from datetime import datetime + from uuid import UUID + + from opensearchpy import OpenSearch + FED_DATASETS_INDEX = "fed-datasets" FED_CAPTURES_INDEX = "fed-captures" diff --git a/gateway/sds_gateway/api_methods/federation/reindex.py b/gateway/sds_gateway/api_methods/federation/reindex.py index 60249bc82..66c14360b 100644 --- a/gateway/sds_gateway/api_methods/federation/reindex.py +++ b/gateway/sds_gateway/api_methods/federation/reindex.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Iterable +from typing import TYPE_CHECKING from django.db import transaction from loguru import logger as log @@ -11,12 +11,16 @@ from sds_gateway.api_methods.federation.compile_federated_data import ( capture_in_published_dataset, ) -from sds_gateway.api_methods.federation.compile_federated_data import compile_federated_doc +from sds_gateway.api_methods.federation.compile_federated_data import ( + compile_federated_doc, +) from sds_gateway.api_methods.federation.compile_federated_data import fed_doc_exists +from sds_gateway.api_methods.federation.compile_federated_data import ( + federation_site_name, +) from sds_gateway.api_methods.federation.compile_federated_data import ( get_federated_export_doc_by_uuid, ) -from sds_gateway.api_methods.federation.compile_federated_data import federation_site_name from sds_gateway.api_methods.federation.events import publish_federation_event from sds_gateway.api_methods.federation.fed_index import LocalFederatedIndexer from sds_gateway.api_methods.models import Capture @@ -24,6 +28,9 @@ from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.utils.opensearch_client import get_opensearch_client +if TYPE_CHECKING: + from collections.abc import Iterable + def reindex_federated_asset( instance: Dataset | Capture, @@ -135,7 +142,9 @@ def _fed_doc_shows_is_deleted( def dataset_needs_federation_reindex(dataset: Dataset) -> bool: if dataset.is_deleted: - return not _fed_doc_shows_is_deleted(dataset, ItemType.DATASET) and fed_doc_exists( + return not _fed_doc_shows_is_deleted( + dataset, ItemType.DATASET + ) and fed_doc_exists( asset_type=ItemType.DATASET, site_name=federation_site_name(), instance=dataset, @@ -154,7 +163,9 @@ def dataset_needs_federation_reindex(dataset: Dataset) -> bool: def capture_needs_federation_reindex(capture: Capture) -> bool: if capture.is_deleted: - return not _fed_doc_shows_is_deleted(capture, ItemType.CAPTURE) and fed_doc_exists( + return not _fed_doc_shows_is_deleted( + capture, ItemType.CAPTURE + ) and fed_doc_exists( asset_type=ItemType.CAPTURE, site_name=federation_site_name(), instance=capture, diff --git a/gateway/sds_gateway/api_methods/federation/signals.py b/gateway/sds_gateway/api_methods/federation/signals.py index a30b09cb7..776d8adc8 100644 --- a/gateway/sds_gateway/api_methods/federation/signals.py +++ b/gateway/sds_gateway/api_methods/federation/signals.py @@ -6,8 +6,12 @@ from django.dispatch import receiver from sds_gateway.api_methods.federation.availability import is_federation_operational -from sds_gateway.api_methods.federation.reindex import schedule_federation_capture_reindex -from sds_gateway.api_methods.federation.reindex import schedule_federation_dataset_reindex +from sds_gateway.api_methods.federation.reindex import ( + schedule_federation_capture_reindex, +) +from sds_gateway.api_methods.federation.reindex import ( + schedule_federation_dataset_reindex, +) from sds_gateway.api_methods.models import Capture from sds_gateway.api_methods.models import Dataset @@ -16,7 +20,7 @@ def federation_dataset_changed( sender: type[Dataset], instance: Dataset, - created: bool, # noqa: FBT001, ARG001 + created: bool, # noqa: FBT001 **kwargs, ) -> None: if not is_federation_operational(): diff --git a/gateway/sds_gateway/api_methods/helpers/extract_drf_metadata.py b/gateway/sds_gateway/api_methods/helpers/extract_drf_metadata.py index 39d08d862..de6fcaa78 100644 --- a/gateway/sds_gateway/api_methods/helpers/extract_drf_metadata.py +++ b/gateway/sds_gateway/api_methods/helpers/extract_drf_metadata.py @@ -160,7 +160,7 @@ def validate_metadata_by_channel( T = typing.TypeVar("T", int, bool) -def convert_or_warn( +def convert_or_warn[T: (int, bool)]( *, value: typing.Any, target_type: type[T], diff --git a/gateway/sds_gateway/api_methods/models.py b/gateway/sds_gateway/api_methods/models.py index dba75d136..280712b56 100644 --- a/gateway/sds_gateway/api_methods/models.py +++ b/gateway/sds_gateway/api_methods/models.py @@ -1004,49 +1004,6 @@ class Dataset(BaseModel): def __str__(self) -> str: return self.name - def is_federation_exportable(self) -> bool: - return ( - not self.is_deleted - and self.is_public - and self.status == DatasetStatus.FINAL - ) - - def _published_snapshot_from_db(self) -> tuple[str, bool] | None: - if not self.pk: - return None - row = ( - type(self).objects.filter(pk=self.pk) - .values_list("status", "is_public") - .first() - ) - return row # (status, is_public) or None - - @staticmethod - def _was_final_status(status: DatasetStatus) -> bool: - return status == DatasetStatus.FINAL - - @staticmethod - def _was_public(is_public: bool) -> bool: - return is_public - - def clean(self) -> None: - super().clean() - snapshot = self._published_snapshot_from_db() - if snapshot is None: - return - - old_status, old_is_public = snapshot - if not self._was_final_status(old_status) and not self._was_public( - is_public=old_is_public, - ): - return - - if self._was_final_status(status=old_status) and self.status != old_status: - raise ValidationError("Final datasets cannot be reverted to draft.") - - if self._was_public(is_public=old_is_public) and self.is_public != old_is_public: - raise ValidationError("Public datasets cannot be reverted to private.") - def save(self, *args, **kwargs) -> None: # Serialize the list fields to a JSON string before saving for field in self.list_fields: @@ -1084,6 +1041,54 @@ def save(self, *args, **kwargs) -> None: self.full_clean() super().save(*args, **kwargs) + def clean(self) -> None: + super().clean() + snapshot = self._published_snapshot_from_db() + if snapshot is None: + return + + old_status, old_is_public = snapshot + if not self._was_final_status(old_status) and not self._was_public( + is_public=old_is_public, + ): + return + + if self._was_final_status(status=old_status) and self.status != old_status: + msg = "Final datasets cannot be reverted to draft." + raise ValidationError(msg) + + if ( + self._was_public(is_public=old_is_public) + and self.is_public != old_is_public + ): + msg = "Public datasets cannot be reverted to private." + raise ValidationError(msg) + + def is_federation_exportable(self) -> bool: + return ( + not self.is_deleted + and self.is_public + and self.status == DatasetStatus.FINAL + ) + + def _published_snapshot_from_db(self) -> tuple[str, bool] | None: + if not self.pk: + return None + return ( + type(self) + .objects.filter(pk=self.pk) + .values_list("status", "is_public") + .first() + ) + + @staticmethod + def _was_final_status(status: DatasetStatus) -> bool: + return status == DatasetStatus.FINAL + + @staticmethod + def _was_public(*, is_public: bool) -> bool: + return is_public + def soft_delete(self) -> None: """Soft delete this record after checking for blockers.""" from sds_gateway.api_methods.utils.asset_access_control import ( # noqa: PLC0415 diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_events.py b/gateway/sds_gateway/api_methods/tests/test_federation_events.py index 14f72943f..363b1f199 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_events.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_events.py @@ -16,7 +16,11 @@ from sds_gateway.api_methods.federation.redis_channel import ( resolve_federation_events_channel, ) +from sds_gateway.api_methods.federation.signals import federation_dataset_changed +from sds_gateway.api_methods.models import Dataset +from sds_gateway.api_methods.models import DatasetStatus from sds_gateway.api_methods.models import ItemType +from sds_gateway.api_methods.tests.factories import DatasetFactory pytestmark = pytest.mark.django_db @@ -100,18 +104,15 @@ class TestFederationSignals: ) @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") + @patch( + "sds_gateway.api_methods.federation.reindex.get_opensearch_client", + new=MagicMock(), + ) def test_dataset_post_save_indexes_when_published( self, - _mock_os: MagicMock, mock_indexer_cls: MagicMock, mock_publish: MagicMock, ) -> None: - from sds_gateway.api_methods.models import Dataset - from sds_gateway.api_methods.models import DatasetStatus - from sds_gateway.api_methods.federation.signals import federation_dataset_changed - from sds_gateway.api_methods.tests.factories import DatasetFactory - mock_indexer = mock_indexer_cls.return_value dataset = DatasetFactory( status=DatasetStatus.FINAL, diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export.py b/gateway/sds_gateway/api_methods/tests/test_federation_export.py index bcabcc96a..4d0318bf1 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export.py @@ -8,11 +8,11 @@ from rest_framework import status from rest_framework.test import APITestCase -from sds_gateway.api_methods.models import DatasetStatus -from sds_gateway.api_methods.models import KeySources from sds_gateway.api_methods.federation.compile_federated_data import ( compile_federated_dataset_doc, ) +from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.models import KeySources from sds_gateway.api_methods.tests.factories import CaptureFactory from sds_gateway.api_methods.tests.factories import DatasetFactory from sds_gateway.users.models import UserAPIKey diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py index 3e4a93e2a..33d8bde47 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_export_contract.py @@ -8,13 +8,13 @@ import pytest from django.contrib.auth import get_user_model -from sds_gateway.api_methods.federation.export_contract import assert_field_names_match from sds_gateway.api_methods.federation.compile_federated_data import ( compile_federated_capture_doc, ) from sds_gateway.api_methods.federation.compile_federated_data import ( compile_federated_dataset_doc, ) +from sds_gateway.api_methods.federation.export_contract import assert_field_names_match from sds_gateway.api_methods.models import DatasetStatus from sds_gateway.api_methods.serializers.capture_serializers import ( CaptureFederationSerializer, diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py index 695fd64a2..f512459ab 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_hardening.py @@ -6,11 +6,15 @@ from unittest.mock import patch import pytest +from config.settings.base import FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT +from config.settings.base import _federation_export_allowed_cidrs_from_env +from config.settings.base import _parse_cidrs from django.conf import settings from django.contrib.auth import get_user_model from django.test import RequestFactory from django.test import override_settings from django.urls import reverse +from opensearchpy import exceptions as os_exceptions from rest_framework import status from rest_framework.test import APITestCase @@ -26,7 +30,9 @@ from sds_gateway.api_methods.federation.availability import ( refresh_federation_operational_state, ) +from sds_gateway.api_methods.federation.compile_federated_data import fed_doc_exists from sds_gateway.api_methods.models import DatasetStatus +from sds_gateway.api_methods.models import ItemType from sds_gateway.api_methods.models import KeySources from sds_gateway.api_methods.tests.factories import DatasetFactory from sds_gateway.users.models import UserAPIKey @@ -154,15 +160,10 @@ def test_refresh_sets_settings_flags(self) -> None: class TestFederationExportCidrParsing: def test_parse_cidrs_skips_blank_tokens(self) -> None: - from config.settings.base import _parse_cidrs - networks = _parse_cidrs(["10.0.0.0/8", "", " ", "192.168.0.0/16"]) - assert len(networks) == 2 + assert len(networks) == 2 # noqa: PLR2004 def test_coerce_empty_env_tokens_to_defaults(self) -> None: - from config.settings.base import FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT - from config.settings.base import _federation_export_allowed_cidrs_from_env - with patch( "config.settings.base.env.list", return_value=["", " "], @@ -176,7 +177,7 @@ class TestFederationOperationalInit: @patch( "sds_gateway.api_methods.federation.availability." "federation_operational_db_ready", - return_value=False, + new=MagicMock(return_value=False), ) @patch( "sds_gateway.api_methods.federation.availability." @@ -185,7 +186,6 @@ class TestFederationOperationalInit: def test_initialize_skips_db_when_tables_missing( self, mock_refresh: MagicMock, - _mock_db_ready: MagicMock, ) -> None: initialize_federation_operational_state() mock_refresh.assert_not_called() @@ -259,23 +259,19 @@ def test_export_returns_503_when_not_operational(self) -> None: class TestFederationOpenSearchReads: - @patch("sds_gateway.api_methods.federation.compile_federated_data.get_opensearch_client") + @patch( + "sds_gateway.api_methods.federation.compile_federated_data.get_opensearch_client" + ) def test_fed_doc_lookup_treats_cluster_errors_as_missing( self, mock_get_client: MagicMock, ) -> None: - from opensearchpy import exceptions as os_exceptions - - from sds_gateway.api_methods.federation.compile_federated_data import ( - fed_doc_exists, - ) - from sds_gateway.api_methods.models import ItemType - from sds_gateway.api_methods.tests.factories import DatasetFactory - - mock_get_client.return_value.get.side_effect = os_exceptions.OpenSearchException( - 503, - "connection failed", - {}, + mock_get_client.return_value.get.side_effect = ( + os_exceptions.OpenSearchException( + 503, + "connection failed", + {}, + ) ) dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) diff --git a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py index 325e5d674..f1c150367 100644 --- a/gateway/sds_gateway/api_methods/tests/test_federation_signals.py +++ b/gateway/sds_gateway/api_methods/tests/test_federation_signals.py @@ -7,6 +7,7 @@ from unittest.mock import patch import pytest +from django.db import transaction from django.test import TestCase from django.test import override_settings @@ -24,6 +25,8 @@ pytestmark = pytest.mark.django_db +_DATASET_AND_CAPTURE = 2 + @contextmanager def _federation_on_commit(): @@ -39,10 +42,12 @@ def _federation_on_commit(): class TestFederationDatasetSignals(TestCase): @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") + @patch( + "sds_gateway.api_methods.federation.reindex.get_opensearch_client", + new=MagicMock(), + ) def test_published_dataset_upserts_and_syncs_captures( self, - _mock_os: MagicMock, mock_indexer_cls: MagicMock, mock_publish: MagicMock, ) -> None: @@ -58,30 +63,29 @@ def test_published_dataset_upserts_and_syncs_captures( created=False, ) - assert mock_indexer.apply_local_event.call_count == 2 + assert mock_indexer.apply_local_event.call_count == _DATASET_AND_CAPTURE item_types = { - c.kwargs["item_type"] - for c in mock_indexer.apply_local_event.call_args_list + c.kwargs["item_type"] for c in mock_indexer.apply_local_event.call_args_list } assert item_types == {ItemType.DATASET, ItemType.CAPTURE} - assert mock_publish.call_count == 2 + assert mock_publish.call_count == _DATASET_AND_CAPTURE @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") + @patch( + "sds_gateway.api_methods.federation.reindex.get_opensearch_client", + new=MagicMock(), + ) @patch( "sds_gateway.api_methods.federation.reindex.get_federated_export_doc_by_uuid", - return_value={"is_deleted": False}, + new=MagicMock(return_value={"is_deleted": False}), ) @patch( "sds_gateway.api_methods.federation.reindex.fed_doc_exists", - return_value=True, + new=MagicMock(return_value=True), ) def test_deleted_dataset_reindexes_full_body( self, - _mock_exists: MagicMock, - _mock_doc: MagicMock, - _mock_os: MagicMock, mock_indexer_cls: MagicMock, mock_publish: MagicMock, ) -> None: @@ -122,16 +126,14 @@ def test_draft_dataset_skips_federation( @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") @patch( "sds_gateway.api_methods.federation.reindex.get_federated_export_doc_by_uuid", - return_value={"is_deleted": True}, + new=MagicMock(return_value={"is_deleted": True}), ) @patch( "sds_gateway.api_methods.federation.reindex.fed_doc_exists", - return_value=True, + new=MagicMock(return_value=True), ) def test_deleted_dataset_skips_when_fed_doc_already_deleted( self, - _mock_exists: MagicMock, - _mock_doc: MagicMock, mock_reindex: MagicMock, ) -> None: dataset = DatasetFactory( @@ -152,10 +154,12 @@ def test_deleted_dataset_skips_when_fed_doc_already_deleted( class TestFederationCaptureSignals(TestCase): @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") + @patch( + "sds_gateway.api_methods.federation.reindex.get_opensearch_client", + new=MagicMock(), + ) def test_capture_on_published_dataset_upserts( self, - _mock_os: MagicMock, mock_indexer_cls: MagicMock, mock_publish: MagicMock, ) -> None: @@ -181,11 +185,10 @@ def test_capture_on_published_dataset_upserts( @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") @patch( "sds_gateway.api_methods.federation.reindex.fed_doc_exists", - return_value=False, + new=MagicMock(return_value=False), ) def test_capture_on_draft_only_skips_without_fed_doc( self, - _mock_exists: MagicMock, mock_reindex: MagicMock, ) -> None: dataset = DatasetFactory(status=DatasetStatus.DRAFT, is_public=False) @@ -197,22 +200,21 @@ def test_capture_on_draft_only_skips_without_fed_doc( @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") + @patch( + "sds_gateway.api_methods.federation.reindex.get_opensearch_client", + new=MagicMock(), + ) @patch( "sds_gateway.api_methods.federation.reindex.get_federated_export_doc_by_uuid", - return_value={"is_deleted": False}, + new=MagicMock(return_value={"is_deleted": False}), ) @patch( "sds_gateway.api_methods.federation.reindex.fed_doc_exists", - return_value=True, + new=MagicMock(return_value=True), ) def test_deleted_capture_reindexes_with_is_deleted( self, - _mock_exists: MagicMock, - _mock_doc: MagicMock, - _mock_os: MagicMock, mock_indexer_cls: MagicMock, - _mock_publish: MagicMock, ) -> None: mock_indexer = mock_indexer_cls.return_value capture = CaptureFactory(is_deleted=True) @@ -230,17 +232,17 @@ def test_deleted_capture_reindexes_with_is_deleted( class TestDatasetDisconnectReindex(TestCase): @patch("sds_gateway.api_methods.federation.reindex.publish_federation_event") @patch("sds_gateway.api_methods.federation.reindex.LocalFederatedIndexer") - @patch("sds_gateway.api_methods.federation.reindex.get_opensearch_client") + @patch( + "sds_gateway.api_methods.federation.reindex.get_opensearch_client", + new=MagicMock(), + ) @patch( "sds_gateway.api_methods.federation.reindex.fed_doc_exists", - return_value=True, + new=MagicMock(return_value=True), ) def test_disconnect_captures_reindexes_orphans( self, - _mock_exists: MagicMock, - _mock_os: MagicMock, mock_indexer_cls: MagicMock, - _mock_publish: MagicMock, ) -> None: mock_indexer = mock_indexer_cls.return_value dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) @@ -259,11 +261,10 @@ def test_disconnect_captures_reindexes_orphans( @patch("sds_gateway.api_methods.federation.reindex.reindex_federated_asset") @patch( "sds_gateway.api_methods.federation.reindex.fed_doc_exists", - return_value=True, + new=MagicMock(return_value=True), ) def test_dataset_soft_delete_reindexes_orphan_capture( self, - _mock_exists: MagicMock, mock_reindex: MagicMock, ) -> None: dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) @@ -294,14 +295,13 @@ def test_rollback_skips_federation_reindex( self, mock_reindex: MagicMock, ) -> None: - from django.db import transaction - dataset = DatasetFactory(status=DatasetStatus.FINAL, is_public=True) try: with transaction.atomic(): dataset.name = "rolled back" dataset.save() - raise RuntimeError("abort") + abort_msg = "abort" + raise RuntimeError(abort_msg) # noqa: TRY301 except RuntimeError: pass mock_reindex.assert_not_called() diff --git a/gateway/sds_gateway/api_methods/tests/test_file_endpoints.py b/gateway/sds_gateway/api_methods/tests/test_file_endpoints.py index ea5c43cea..afa6218e6 100644 --- a/gateway/sds_gateway/api_methods/tests/test_file_endpoints.py +++ b/gateway/sds_gateway/api_methods/tests/test_file_endpoints.py @@ -699,7 +699,9 @@ def test_detach_file_from_datasets(self) -> None: assert not self.file.datasets.exists() def test_detach_file_keeps_published_dataset_links(self) -> None: - draft = DatasetFactory(owner=self.user, status=DatasetStatus.DRAFT, is_public=False) + draft = DatasetFactory( + owner=self.user, status=DatasetStatus.DRAFT, is_public=False + ) published = DatasetFactory( owner=self.user, status=DatasetStatus.FINAL, @@ -714,5 +716,7 @@ def test_detach_file_keeps_published_dataset_links(self) -> None: assert response.status_code == status.HTTP_200_OK self.file.refresh_from_db() - assert list(self.file.datasets.values_list("uuid", flat=True)) == [published.uuid] + assert list(self.file.datasets.values_list("uuid", flat=True)) == [ + published.uuid + ] assert self.file.dataset is None diff --git a/gateway/sds_gateway/api_methods/utils/relationship_utils.py b/gateway/sds_gateway/api_methods/utils/relationship_utils.py index 6350fc7e9..b8041df5f 100644 --- a/gateway/sds_gateway/api_methods/utils/relationship_utils.py +++ b/gateway/sds_gateway/api_methods/utils/relationship_utils.py @@ -312,10 +312,7 @@ def _datasets_removable_from_item(item: Capture | File) -> QuerySet[Dataset]: linked = get_capture_datasets(item, include_deleted=False) else: linked = get_file_datasets(item, include_deleted=False) - return linked.exclude( - Q(status=DatasetStatus.FINAL) | - Q(is_public=True) - ) + return linked.exclude(Q(status=DatasetStatus.FINAL) | Q(is_public=True)) @transaction.atomic diff --git a/gateway/sds_gateway/api_methods/views/federation_endpoints.py b/gateway/sds_gateway/api_methods/views/federation_endpoints.py index d02192a13..f51816552 100644 --- a/gateway/sds_gateway/api_methods/views/federation_endpoints.py +++ b/gateway/sds_gateway/api_methods/views/federation_endpoints.py @@ -40,11 +40,11 @@ def _federation_export_uuid(pk: str | None) -> UUID: if not pk: - raise NotFound() + raise NotFound try: return UUID(pk) except ValueError as exc: - raise NotFound() from exc + raise NotFound from exc @extend_schema(exclude=True) @@ -81,7 +81,7 @@ def export_dataset_detail( asset_uuid=asset_uuid, ) if body is None: - raise NotFound() + raise NotFound return Response(body) @action(detail=False, methods=["get"], url_path="export/captures") @@ -107,5 +107,5 @@ def export_capture_detail( asset_uuid=asset_uuid, ) if body is None: - raise NotFound() + raise NotFound return Response(body) diff --git a/gateway/sds_gateway/users/backend_service_key_utils.py b/gateway/sds_gateway/users/backend_service_key_utils.py index 8501c119a..167ff217e 100644 --- a/gateway/sds_gateway/users/backend_service_key_utils.py +++ b/gateway/sds_gateway/users/backend_service_key_utils.py @@ -51,7 +51,7 @@ def replace_user_api_key_for_source( name: str, description: str = "", ) -> str: - """Delete existing keys for ``(user, source)`` and mint a new Api-Key (SVI pattern).""" + """Delete existing keys for ``(user, source)`` and mint a new Api-Key.""" UserAPIKey.objects.filter(user=user, source=source).delete() _obj, raw_key = UserAPIKey.objects.create_key( name=name, @@ -91,7 +91,7 @@ def update_svi_server_token() -> None: def update_federation_sync_drf_token() -> None: - """Sync federation-sync DRF ``Token`` from ``FEDERATION_SYNC_DRF_TOKEN`` (shared env).""" + """Sync federation-sync DRF ``Token`` from ``FEDERATION_SYNC_DRF_TOKEN``.""" token_key = settings.FEDERATION_SYNC_DRF_TOKEN if len(token_key) != settings.TOKEN_LENGTH: msg = ( diff --git a/gateway/sds_gateway/users/tests/test_svi_utils.py b/gateway/sds_gateway/users/tests/test_svi_utils.py index fb66142ee..e79e97aa2 100644 --- a/gateway/sds_gateway/users/tests/test_svi_utils.py +++ b/gateway/sds_gateway/users/tests/test_svi_utils.py @@ -5,8 +5,8 @@ from django.db.utils import IntegrityError from rest_framework.authtoken.models import Token -from sds_gateway.users.models import User from sds_gateway.users.backend_service_key_utils import update_svi_server_token +from sds_gateway.users.models import User @pytest.mark.django_db