Scaffolding: Federation gateway setup

.gitignore

@@ -6,3 +6,4 @@
 **/agents.md
 agents.md
 htmlcov/
+federation-shared.env

federation-shared.example.env

@@ -0,0 +1,7 @@
+# Shared by gateway + federation sync (repo root; gateway/scripts/generate-secrets.sh).
+# Compose: env_file → ../../federation-shared.env (gateway) or ../federation-shared.env (federation).
+# FEDERATION_SYNC_DRF_TOKEN: bootstrap Token for get-federation-sync-api-key.
+# FEDERATION_SYNC_SERVER_API_KEY: export Api-Key (sync service writes after mint).
+
+FEDERATION_SYNC_DRF_TOKEN=
+FEDERATION_SYNC_SERVER_API_KEY=

gateway/.envs/example/django.env

@@ -8,6 +8,24 @@ HOSTNAME=localhost:8000
 API_VERSION=v1
 API_KEY=
 
+# FEDERATION
+# ------------------------------------------------------------------------------
+# Load federation-shared.env from repo root on gateway + sync compose stacks (generate-secrets).
+# prepare_gateway runs init_federation_sync_token (DB Token from FEDERATION_SYNC_DRF_TOKEN).
+# Sync startup mints FEDERATION_SYNC_SERVER_API_KEY via get-federation-sync-api-key.
+# Set FEDERATION_SITE_NAME (e.g. crc) when enabling federation; use SDS_SITE_FQDN for the public host (RFC [site].fqdn).
+# Master switch for export APIs and Redis federation events.
+# FEDERATION_ENABLED=true
+# RFC [site].name (short peer id); set SDS_SITE_FQDN separately for [site].fqdn.
+# FEDERATION_SITE_NAME=crc
+# Redis pub/sub channel federation-sync subscribes to.
+# FEDERATION_EVENTS_CHANNEL=federation:events
+# Health probe target (federation-sync service).
+# FEDERATION_SYNC_HEALTH_URL=http://federation-sync:8000/sync/health
+# FEDERATION_SYNC_USER_EMAIL=federation-sync@internal.local
+# Comma-separated CIDRs allowed to call export (default: private Docker ranges).
+# FEDERATION_EXPORT_ALLOWED_CIDRS=
+
 # AUTH0
 # ------------------------------------------------------------------------------
 # Set these from your Auth0 application settings

gateway/compose.local.yaml

@@ -83,6 +83,7 @@ services:
             - ./.envs/local/storage.env
             - ./.envs/local/postgres.env
             - ./.envs/local/opensearch.env
+            - ../federation-shared.env
         environment:
             - SKIP_POSTGRES_WAIT=1
         # remember /entrypoint runs first

gateway/config/api_router.py

@@ -5,6 +5,7 @@
 from sds_gateway.api_methods.views.auth_endpoints import ValidateAuthViewSet
 from sds_gateway.api_methods.views.capture_endpoints import CaptureViewSet
 from sds_gateway.api_methods.views.dataset_endpoints import DatasetViewSet
+from sds_gateway.api_methods.views.federation_endpoints import FederationViewSet
 from sds_gateway.api_methods.views.file_endpoints import FileViewSet
 from sds_gateway.api_methods.views.file_endpoints import check_contents_exist
 from sds_gateway.users.api.views import UserViewSet
@@ -17,6 +18,7 @@
 router.register(r"assets/files", FileViewSet, basename="files")
 router.register(r"assets/captures", CaptureViewSet, basename="captures")
 router.register(r"assets/datasets", DatasetViewSet, basename="datasets")
+router.register(r"federation", FederationViewSet, basename="federation")
 
 if settings.VISUALIZATIONS_ENABLED:
     router.register(r"visualizations", VisualizationViewSet, basename="visualizations")

gateway/config/settings/base.py

@@ -1,6 +1,7 @@
 """Base settings to build other settings files upon."""
 # ruff: noqa: ERA001
 
+import ipaddress
 import random
 import string
 from pathlib import Path
@@ -9,6 +10,9 @@
 
 from celery.schedules import crontab
 from environs import env
+from sds_gateway.api_methods.federation.redis_channel import (
+    resolve_federation_events_channel,
+)
 
 from config.settings.logs import ColoredFormatter
 from config.settings.utils import guess_admin_console_env
@@ -24,6 +28,41 @@ def __get_random_token(length: int) -> str:
     )
 
 
+FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT: list[str] = [
+    "127.0.0.1/32",
+    "::1/128",
+    "10.0.0.0/8",
+    "172.16.0.0/12",
+    "192.168.0.0/16",
+]
+
+
+def _parse_cidrs(
+    raw: list[str],
+) -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]:
+    networks: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = []
+    for item in raw:
+        text = str(item).strip()
+        if not text:
+            continue
+        networks.append(ipaddress.ip_network(text, strict=False))
+    return networks
+
+
+def _federation_export_allowed_cidrs_from_env() -> list[
+    ipaddress.IPv4Network | ipaddress.IPv6Network
+]:
+    raw = env.list(
+        "FEDERATION_EXPORT_ALLOWED_CIDRS",
+        default=FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT,
+    )
+    tokens = [str(item).strip() for item in raw]
+    tokens = [token for token in tokens if token]
+    if not tokens:
+        tokens = list(FEDERATION_EXPORT_ALLOWED_CIDRS_DEFAULT)
+    return _parse_cidrs(tokens)
+
+
 env.read_env()
 
 SITE_DOMAIN: str = env.str("SITE_DOMAIN", default="localhost:8000")
@@ -610,7 +649,10 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str:
         "rest_framework.authentication.SessionAuthentication",
         "sds_gateway.api_methods.authentication.APIKeyAuthentication",
     ),
-    "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",),
+    "DEFAULT_PERMISSION_CLASSES": (
+        "rest_framework.permissions.IsAuthenticated",
+        "sds_gateway.api_methods.permissions.DisallowFederationSyncKey",
+    ),
     "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema",
     "DEFAULT_THROTTLE_RATES": {
         "vis_stream": VIS_STREAM_THROTTLE_RATE,
@@ -709,6 +751,56 @@ def _strip_endpoint_scheme(endpoint_url: str) -> str:
 SDS_PROGRAMMATIC_SITE_NAME: str = env.str("SDS_PROGRAMMATIC_SITE_NAME", default="sds")
 SDS_SITE_FQDN: str = env.str("SDS_SITE_FQDN", default="localhost")
 
+# Federation peer short name (RFC [site].name, e.g. crc, haystack);
+# not SDS_PROGRAMMATIC_SITE_NAME.
+FEDERATION_SITE_NAME: str = env.str("FEDERATION_SITE_NAME", default="").strip()
+# Master switch: when False, federation export and Redis events are inactive.
+FEDERATION_ENABLED: bool = env.bool("FEDERATION_ENABLED", default=False)
+_federation_events_channel_override: str = env.str(
+    "FEDERATION_EVENTS_CHANNEL",
+    default="",
+).strip()
+FEDERATION_EVENTS_CHANNEL: str = resolve_federation_events_channel(
+    site_name=FEDERATION_SITE_NAME,
+    channel_override=_federation_events_channel_override,
+)
+FEDERATION_SYNC_USER_EMAIL: str = env.str(
+    "FEDERATION_SYNC_USER_EMAIL",
+    default="federation-sync@internal.local",
+)
+# From federation-shared.env (compose env_file);
+# used by init_federation_sync_token only.
+FEDERATION_SYNC_DRF_TOKEN: str = env.str("FEDERATION_SYNC_DRF_TOKEN", default="")
+FEDERATION_SYNC_HEALTH_URL: str = env.str(
+    "FEDERATION_SYNC_HEALTH_URL",
+    default="http://federation-sync:8000/sync/health",
+)
+FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT: float = env.float(
+    "FEDERATION_SYNC_HEALTH_PROBE_TIMEOUT",
+    default=2.0,
+)
+FEDERATION_SKIP_SYNC_HEALTH_PROBE: bool = env.bool(
+    "FEDERATION_SKIP_SYNC_HEALTH_PROBE",
+    default=False,
+)
+FEDERATION_SKIP_SYNC_API_KEY_CHECK: bool = env.bool(
+    "FEDERATION_SKIP_SYNC_API_KEY_CHECK",
+    default=False,
+)
+FEDERATION_SKIP_REDIS_PROBE: bool = env.bool(
+    "FEDERATION_SKIP_REDIS_PROBE",
+    default=False,
+)
+# Set at startup / periodic recheck by federation.availability.
+FEDERATION_OPERATIONAL: bool = False
+FEDERATION_OPERATIONAL_REASON: str = ""
+# Tests may set via override_settings without running probes.
+FEDERATION_OPERATIONAL_OVERRIDE: bool | None = None
+# Export API: internal Docker/private networks (sync → django on sds-network).
+FEDERATION_EXPORT_ALLOWED_CIDRS: list[ipaddress.IPv4Network | ipaddress.IPv6Network] = (
+    _federation_export_allowed_cidrs_from_env()
+)
+
 # ADMIN_CONSOLE_ENV is used to visually distinguish between different environments
 # (production, staging, local) in the admin console and error emails. It does not affect
 # any functionality and it is meant to prevent changes in production meant for testing

gateway/pyproject.toml

@@ -147,6 +147,10 @@
 
     # https://deptry.com/usage/#per-rule-ignores
     [tool.deptry.per_rule_ignores]
+        DEP001 = [
+            # optional monorepo sibling; contract tests add federation/ to sys.path
+            "sds_federation",
+        ]
         DEP002 = [
             # packages that are installed but not imported
             "argon2-cffi",                # used by django for argon2 password hashing

gateway/scripts/fallow-cross-file-dupes.sh

@@ -2,24 +2,31 @@
 set -euo pipefail
 cd "$(dirname "$0")/.."
 
-# Use first available: vpx > bunx > npx
-if command -v pnpx &>/dev/null; then
-	RUNNER=pnpx
-elif command -v vpx &>/dev/null; then
-	RUNNER=vpx
-elif command -v bunx &>/dev/null; then
-	RUNNER=bunx
-elif command -v npx &>/dev/null; then
-	RUNNER=npx
+LOCAL_FALLOW="node_modules/.bin/fallow"
+if [[ -x "${LOCAL_FALLOW}" ]]; then
+	"${LOCAL_FALLOW}" dupes --format json -q | jq -e '
+  [ (.clone_groups // .dupes.clone_groups // [])[]
+    | select((.instances | map(.file) | unique | length) > 1)
+  ] | length == 0
+' >/dev/null
 else
-	echo "Error: neither vpx, bunx, nor npx found in PATH" >&2
-	exit 1
-fi
+	# Use first available: vpx > bunx > npx (avoid pnpx; global pnpm may need newer Node)
+	if command -v vpx &>/dev/null; then
+		RUNNER=(vpx)
+	elif command -v bunx &>/dev/null; then
+		RUNNER=(bunx)
+	elif command -v npx &>/dev/null; then
+		RUNNER=(npx)
+	else
+		echo "Error: neither local fallow, vpx, bunx, nor npx found" >&2
+		exit 1
+	fi
 
-"${RUNNER}" fallow dupes --format json -q | jq -e '
+	"${RUNNER[@]}" fallow dupes --format json -q | jq -e '
   [ (.clone_groups // .dupes.clone_groups // [])[]
     | select((.instances | map(.file) | unique | length) > 1)
   ] | length == 0
 ' >/dev/null
+fi
 
 echo "No cross-file clone groups detected."

gateway/scripts/generate-secrets.sh

@@ -3,8 +3,12 @@ set -Eeuo pipefail
 
 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 GATEWAY_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd)
-SFS_ROOT=$(cd "${GATEWAY_ROOT}/../seaweedfs" && pwd)
+SDS_ROOT=$(cd "${GATEWAY_ROOT}/.." && pwd)
+FEDERATION_ROOT=$(cd "${SDS_ROOT}/federation" && pwd)
+SFS_ROOT=$(cd "${SDS_ROOT}/seaweedfs" && pwd)
 EXAMPLE_DIR="${GATEWAY_ROOT}/.envs/example"
+FEDERATION_SHARED_TEMPLATE="${SDS_ROOT}/federation-shared.example.env"
+FEDERATION_SHARED_OUTPUT="${SDS_ROOT}/federation-shared.env"
 
 # PRIMARY (RustFS or SeaweedFS)
 PRIMARY_ACCESS_KEY_ID=""
@@ -16,6 +20,20 @@ PRIMARY_S3_ENDPOINT_URL=""
 SECONDARY_ACCESS_KEY_ID=""
 SECONDARY_SECRET_ACCESS_KEY=""
 
+FEDERATION_SYNC_DRF_TOKEN=""
+
+function ensure_federation_sync_drf_token() {
+	local env_type="$1"
+	if [[ -n "${FEDERATION_SYNC_DRF_TOKEN}" ]]; then
+		return 0
+	fi
+	if [[ "${env_type}" == "ci" ]]; then
+		FEDERATION_SYNC_DRF_TOKEN="ci-federation-sync-drf-01234567890123456"
+	else
+		FEDERATION_SYNC_DRF_TOKEN=$(generate_secret 40)
+	fi
+}
+
 function usage() {
 	cat <<EOF
 Usage: ${0} [OPTIONS] <local|production|ci>
@@ -35,7 +53,8 @@ EXAMPLES:
     ${0} production         # Generate production env files
 
     NOTES:
-        - Generated files are placed in .envs/<env>/ directory
+        - Generated gateway files are placed in gateway/.envs/<env>/
+        - federation-shared.env is written to the repo root (from federation-shared.example.env)
         - Example templates are read from .envs/example/
         - Secrets are randomly generated using OpenSSL
         - CI environment uses insecure but deterministic values for ephemeral usage
@@ -120,7 +139,7 @@ function process_env_file() {
 		return 0
 	fi
 
-	echo "  ✓  Generating ${output}"
+	echo "  ✓  Generating ${output} (from ${template})"
 
 	local content
 	content=$(cat "${template}")
@@ -167,6 +186,11 @@ function process_env_file() {
 	# set WEB_CONCURRENCY based on CPU cores (applies to all environments)
 	content="${content//WEB_CONCURRENCY=4/WEB_CONCURRENCY=${web_concurrency}}"
 
+	if [[ "${filename}" == "federation-shared.example.env" ]]; then
+		ensure_federation_sync_drf_token "${env_type}"
+		content="${content//FEDERATION_SYNC_DRF_TOKEN=/FEDERATION_SYNC_DRF_TOKEN=${FEDERATION_SYNC_DRF_TOKEN}}"
+	fi
+
 	if [[ "${filename}" == "storage.env" ]]; then
 		# PRIMARY vars
 		content="${content//PRIMARY_ACCESS_KEY_ID=admin/PRIMARY_ACCESS_KEY_ID=${PRIMARY_ACCESS_KEY_ID}}"
@@ -232,7 +256,11 @@ function process_env_file() {
 		# content="${content//RUSTFS_SECRET_ACCESS_KEY=<SAME AS SECONDARY_ROOT_PASSWORD>/RUSTFS_SECRET_ACCESS_KEY=${SECONDARY_SECRET_ACCESS_KEY}}"
 	fi
 
-	# write to output
+	# write to output (never overwrite *.example.env templates)
+	if [[ "${output}" == *".example.env" ]]; then
+		echo "ERROR: refused to write secrets to example template: ${output}" >&2
+		exit 1
+	fi
 	mkdir -p "$(dirname "${output}")"
 	echo "${content}" >"${output}"
 	chmod 600 "${output}"
@@ -243,12 +271,16 @@ function set_permissions() {
 	env_dirs=(
 		"${GATEWAY_ROOT}/.envs"
 		"${SFS_ROOT}/.envs"
+		"${FEDERATION_ROOT}/.envs"
 	)
 	for dir in "${env_dirs[@]}"; do
 		if [ -d "${dir}" ]; then
 			find "${dir}" -type f -name "*.env" -exec chmod --changes 600 {} \;
 		fi
 	done
+	if [[ -f "${FEDERATION_SHARED_OUTPUT}" ]]; then
+		chmod --changes 600 "${FEDERATION_SHARED_OUTPUT}"
+	fi
 }
 
 function main() {
@@ -326,10 +358,24 @@ function main() {
 		process_env_file "${template}" "${output}" "${env_type}" "${force}"
 	done
 
+	if [[ -f "${FEDERATION_SHARED_TEMPLATE}" ]]; then
+		# Read federation-shared.example.env → write repo-root federation-shared.env only
+		process_env_file \
+			"${FEDERATION_SHARED_TEMPLATE}" \
+			"${FEDERATION_SHARED_OUTPUT}" \
+			"${env_type}" \
+			"${force}"
+	else
+		echo "  ⏭  ${FEDERATION_SHARED_TEMPLATE} not found (skipping federation-shared.env)"
+	fi
+
 	set_permissions
 
 	echo ""
 	echo "✅ Secrets generated successfully in ${target_dir_gwy}/"
+	if [[ -f "${FEDERATION_SHARED_OUTPUT}" ]]; then
+		echo "   Federation shared env: ${FEDERATION_SHARED_OUTPUT}"
+	fi
 	echo ""
 	echo "Next steps:"
 	if [[ "${env_type}" == "ci" ]]; then