diff --git a/CLAUDE.md b/CLAUDE.md index 03786cc4..a8716d07 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -102,6 +102,8 @@ Writes `manifest.jsonl` (one line per alert) plus `images/{source_api}/{platform **Backend data flow**: Alert API → ingestion scripts → annotation_api DB → frontend UI → human annotations +**Ingestion**: Daily ingestion is connector-driven (worker sweep at 03:00 UTC, configured at `/connectors`); the CLI import script is retained for backfilling arbitrary date ranges. + **Processing stages** (sequence): `IMPORTED` → `READY_TO_ANNOTATE` → `SEQ_ANNOTATION_DONE` → `ANNOTATED`. Two-lane exit: FP-only lanes jump straight to `ANNOTATED` at classify submit; smoke lanes park at `SEQ_ANNOTATION_DONE`, get auto-annotated per alert once every sibling (shared `platform_alert_id`) is classified, and reach `ANNOTATED` via the Smoke Localization submit (see `docs/specs/2026-07-28-smoke-localization-entry-point-design.md`). **Backend patterns**: CRUD modules per entity, Pydantic schemas separate from SQLModel, dependency injection, fastapi-pagination, IoU-based annotation generation service. diff --git a/annotation_api/.env.example b/annotation_api/.env.example index 1a4b58b0..5a0b2a9d 100644 --- a/annotation_api/.env.example +++ b/annotation_api/.env.example @@ -41,3 +41,9 @@ ALERT_API_LOGIN=your_alert_api_username ALERT_API_PASSWORD=your_alert_api_password ALERT_API_ADMIN_LOGIN=your_admin_username ALERT_API_ADMIN_PASSWORD=your_admin_password + +# Fernet key encrypting alert-API connector passwords at rest. +# Generate: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" +CONNECTOR_SECRET_KEY= +# Where the worker reaches the annotation API (compose service name). +ANNOTATION_API_INTERNAL_URL=http://annotation_api:5050 diff --git a/annotation_api/CLAUDE.md b/annotation_api/CLAUDE.md index 9d6b0cea..1110dd0b 100644 --- a/annotation_api/CLAUDE.md +++ b/annotation_api/CLAUDE.md @@ -453,6 +453,38 @@ uv run python -m scripts.data_transfer.ingestion.alert_api.import \ - **Logging support** - Configurable log levels for debugging - **Stage management** - Automatic transitions from alert API data to READY_TO_ANNOTATE stage +### Alert API Connectors + +Ongoing daily ingestion runs through connectors instead of the CLI. A connector is +one alert API credential (base URL, login, password) plus the set of remote +organizations it should import from. + +- **Configuration**: Connectors are managed in the frontend at `/connectors`, + superuser only. Creating one stores the credential, discovers the alert API's + organizations, and lets the superuser enable individual organizations for import. +- **Schedule**: The worker sweeps all enabled connectors daily at 03:00 UTC + (`schedule_connector_imports` in `src/app/worker.py`) and imports each one's + trailing window (`run_connector_import`). `trailing_days` (default 3, configurable + per connector) is both the re-check window and the catch-up mechanism — a worker + that missed a run recovers the lost date inside the next run's window, so no + "already ran today" bookkeeping is needed. +- **`CONNECTOR_SECRET_KEY`**: Required for connectors to work. Alert-API passwords + can't be hashed (the worker needs the plaintext to log in), so they're Fernet- + encrypted at rest with this key. Generate one with: + ```bash + python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" + ``` + If unset, connector create/update returns `400` and the worker skips connector + imports — existing deployments that never set it keep working untouched. Losing + the key means re-entering credentials through the UI. +- **Backfill**: The UI has no backfill and enabling a new organization does not + retroactively import its history. `make import-alert-api DATE_FROM=… DATE_END=…` + remains the way to import an arbitrary date range by hand. +- **Coverage**: Each (connector, organization, day) import attempt is recorded as a + coverage row, rendered as a heatmap on the connector detail page. A day with zero + alerts is recorded as `ok` with zero counts — deliberately distinct from a day + that was never attempted, so a dashed cell always means "we never got there." + ## Troubleshooting ### Common Issues diff --git a/annotation_api/Dockerfile b/annotation_api/Dockerfile index 4230f201..86dface7 100644 --- a/annotation_api/Dockerfile +++ b/annotation_api/Dockerfile @@ -35,6 +35,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ COPY src/alembic.ini /app/alembic.ini COPY src/migrations /app/migrations COPY src/app /app/app +# The worker imports the alert-API importer as a library (see app/worker.py). +COPY scripts /app/scripts # Install project RUN --mount=type=cache,target=/root/.cache/uv \ diff --git a/annotation_api/pyproject.toml b/annotation_api/pyproject.toml index bef90a12..7dc74e62 100644 --- a/annotation_api/pyproject.toml +++ b/annotation_api/pyproject.toml @@ -41,6 +41,9 @@ dependencies = [ "pillow>=10.0.0", "procrastinate>=2.0.0,<4.0.0", "psycopg[binary]>=3.1.0", + "cryptography>=42.0.0", + "pyyaml>=6.0", + "python-dotenv>=1.0.0", ] [dependency-groups] @@ -59,7 +62,6 @@ dev = [ "requests-mock>=1.11.0", "httpx>=0.23.0", "aiosqlite>=0.16.0,<1.0.0", - "pyyaml>=6.0", ] [tool.coverage.run] diff --git a/annotation_api/scripts/data_transfer/ingestion/alert_api/client.py b/annotation_api/scripts/data_transfer/ingestion/alert_api/client.py index 2bb239da..84381b07 100644 --- a/annotation_api/scripts/data_transfer/ingestion/alert_api/client.py +++ b/annotation_api/scripts/data_transfer/ingestion/alert_api/client.py @@ -63,7 +63,13 @@ def api_get(route: str, access_token: str): """ headers = make_request_headers(access_token=access_token) logging.debug(f"Making an HTTP request to route {route}") - response = requests.get(route, headers=headers) + # 30s: these are list endpoints (sequences/cameras/organizations) that can + # legitimately take longer than the 5s token exchange under real load, but + # this now also runs on the worker's connector-verify path behind a button + # a human is watching (via asyncio.to_thread), so it must never hang + # forever on a black-holed connection — no timeout previously bounded this + # call at all. + response = requests.get(route, headers=headers, timeout=30) try: return response.json() except Exception: diff --git a/annotation_api/scripts/data_transfer/ingestion/alert_api/import.py b/annotation_api/scripts/data_transfer/ingestion/alert_api/import.py index 2e4c17a5..b836dd20 100644 --- a/annotation_api/scripts/data_transfer/ingestion/alert_api/import.py +++ b/annotation_api/scripts/data_transfer/ingestion/alert_api/import.py @@ -65,37 +65,19 @@ """ import argparse -import concurrent.futures import logging import os import re import sys -import time from datetime import datetime -from typing import List +from typing import List, Optional from dotenv import load_dotenv from rich.console import Console -from rich.panel import Panel -from rich.progress import ( - Progress, - SpinnerColumn, - TextColumn, - BarColumn, - TaskProgressColumn, -) - -# Import new modular components -from .progress_management import ErrorCollector, StepManager, LogSuppressor -from .worker_config import WorkerConfig -from .sequence_fetching import fetch_all_sequences_within -from . import object_split -from .annotation_management import ( - valid_date, - annotate_split_sequence, -) + +from .annotation_management import valid_date +from .runner import ImportConfig, run_import from . import shared -from . import client as alert_api_client from app.clients import annotation_api load_dotenv() @@ -249,57 +231,23 @@ def validate_args(args: argparse.Namespace) -> bool: return True -def test_annotation_credentials( +def authenticate_annotation_api( base_url: str, login: str, password: str, label: str, console: Console -) -> bool: +) -> Optional[str]: """ Attempt to authenticate against an annotation API endpoint. - """ - try: - annotation_api.get_auth_token(base_url, username=login, password=password) - console.print(f"[green]✅ {label} auth OK[/] [dim]({login}@{base_url})[/]") - return True - except Exception as exc: - console.print(f"[red]❌ {label} auth failed[/]: {exc}") - return False - -def auto_skip_boxless( - annotation_api_url: str, - login: str, - password: str, - source_api: str, - boxless_alert_ids: List[int], - console: Console, - error_collector: ErrorCollector, -) -> dict: - """ - Best-effort auto-skip of boxless alerts (#333): park their zero-object - lanes via the skip overlay. Never raises — a skip failure must not fail - an otherwise successful import. + Returns the access token, or None when authentication failed. """ - counts = {"skipped": 0, "already_skipped": 0, "failed": 0} try: - auth_token = annotation_api.get_auth_token( - annotation_api_url, username=login, password=password - ) - counts = shared.skip_boxless_alerts( - annotation_api_url, auth_token, source_api, boxless_alert_ids + token = annotation_api.get_auth_token( + base_url, username=login, password=password ) + console.print(f"[green]✅ {label} auth OK[/] [dim]({login}@{base_url})[/]") + return token except Exception as exc: - counts["failed"] = len(boxless_alert_ids) - logging.warning("boxless auto-skip aborted: %s", exc) - console.print( - f"[blue]⏭️ Auto-skipped {counts['skipped']} boxless alert(s) " - f"({counts['already_skipped']} already skipped, " - f"{counts['failed']} failed): {boxless_alert_ids}[/]" - ) - if counts["failed"] > 0: - error_collector.add_warning( - f"{counts['failed']} boxless alert(s) could not be auto-skipped; " - "their zero-object lanes remain in the queue." - ) - return counts + console.print(f"[red]❌ {label} auth failed[/]: {exc}") + return None def parse_sequence_selection(sequence_arg: str) -> List[int]: @@ -335,7 +283,7 @@ def parse_sequence_selection(sequence_arg: str) -> List[int]: def main() -> None: - """Main execution function with comprehensive error handling and progress tracking.""" + """Parse argv and the environment into an ImportConfig, then run the import.""" parser = make_cli_parser() args = parser.parse_args() @@ -346,7 +294,6 @@ def main() -> None: level=args.loglevel.upper(), format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) - logger = logging.getLogger(__name__) # bucket-copy derives its source bucket from the French deployment's # config (PLATFORM_SERVER_NAME), so it cannot work against CENIA. @@ -372,38 +319,8 @@ def main() -> None: # Get source_api from alert API URL source_api = get_source_api_from_url(args.alert_api_url) - # Initialize components - worker_config = WorkerConfig(args.max_workers) console = Console() - suppress_logs = args.loglevel != "debug" # Suppress logs unless in debug mode - step_manager = StepManager(console, show_timing=True) - error_collector = ErrorCollector() - - # Initialize comprehensive statistics - stats = { - # Import statistics (Step 1) - "records_fetched": 0, - "sequences_attempted_import": 0, - "sequences_import_successful": 0, - "sequences_import_failed": 0, - "sequences_skipped": 0, - "sequences_refreshed": 0, - "refresh_failures": 0, - "refresh_skipped": 0, - "detections_skipped": 0, - "detections_attempted_import": 0, - "detections_import_successful": 0, - "detections_import_failed": 0, - # Annotation statistics (Step 4) - "total_sequences_for_annotation": 0, - "annotations_successful": 0, - "annotations_failed": 0, - "annotations_created": 0, - "sequences_rolled_back": 0, - } - # Initialize organization early to avoid reference errors in exception handlers - organization = shared.getenv_with_fallback("ALERT_API_LOGIN") or "unknown" selected_sequence_list: List[int] = [] sequence_list_source = "CLI input" @@ -432,7 +349,7 @@ def main() -> None: target_login, target_password = shared.get_annotation_credentials( args.annotation_api_url ) - target_ok = test_annotation_credentials( + annotation_api_token = authenticate_annotation_api( args.annotation_api_url, target_login, target_password, @@ -440,509 +357,38 @@ def main() -> None: console, ) - if not target_ok: + if annotation_api_token is None: console.print("[red]❌ Aborting due to authentication failure[/]") sys.exit(1) - # Print header - console.print() - console.print( - Panel( - "[bold blue]Alert API Data Import & Processing[/]", - title="🔥 Pyronear Data Import", - border_style="blue", - padding=(0, 2), - ) - ) - - if args.loglevel == "debug": - console.print(f"[blue]ℹ️ Date range: {args.date_from} to {args.date_end}[/]") - console.print( - f"[blue]ℹ️ Alert API: {args.alert_api_url} (source_api: {source_api})[/]" - ) - console.print(f"[blue]ℹ️ Worker config: {worker_config}[/]") - - try: - # Step 1: Fetch alert API data - successfully_imported_sequence_ids = [] - step_manager.start_step( - 1, - "Alert API Data Import", - f"Fetching {organization} data from {args.date_from} to {args.date_end} using {worker_config.base_workers} workers", - ) - - if not shared.validate_available_env_variables(): - console.print( - "[red]❌ Missing required environment variables for alert API[/]" - ) - step_manager.complete_step(False, "Missing environment variables") - sys.exit(1) - - # Get alert API credentials - alert_api_login = shared.getenv_with_fallback("ALERT_API_LOGIN") - alert_api_password = shared.getenv_with_fallback("ALERT_API_PASSWORD") - alert_api_admin_login = shared.getenv_with_fallback("ALERT_API_ADMIN_LOGIN") - alert_api_admin_password = shared.getenv_with_fallback( - "ALERT_API_ADMIN_PASSWORD" - ) - - if not all( - [ - alert_api_login, - alert_api_password, - alert_api_admin_login, - alert_api_admin_password, - ] - ): - error_collector.add_error("Missing alert API credentials") - step_manager.complete_step(False, "Missing alert API credentials") - sys.exit(1) - - # Get access tokens with progress display - auth_start_time = time.time() - with console.status( - f"[bold blue]🔐 Authenticating with alert API ({organization})...", - spinner="dots", - ) as status: - try: - status.update(f"[bold blue]🔐 Getting {organization} access token...") - access_token = alert_api_client.get_api_access_token( - api_endpoint=args.alert_api_url, - username=alert_api_login, - password=alert_api_password, - ) - - status.update("[bold blue]🔐 Getting admin access token...") - access_token_admin = alert_api_client.get_api_access_token( - api_endpoint=args.alert_api_url, - username=alert_api_admin_login, - password=alert_api_admin_password, - ) - - auth_duration = time.time() - auth_start_time - console.print( - f"[green]✅ Authentication successful[/] [dim]({auth_duration:.1f}s)[/]" - ) - - except Exception as e: - error_collector.add_error(f"Authentication failed: {e}") - step_manager.complete_step(False, f"Authentication failed: {e}") - sys.exit(1) - - # Fetch alert API records - try: - records = fetch_all_sequences_within( - date_from=args.date_from, - date_end=args.date_end, - detections_limit=args.frames_limit, - detections_order_by="asc", - api_endpoint=args.alert_api_url, - access_token=access_token, - access_token_admin=access_token_admin, - worker_config=worker_config, - selected_sequence_list=selected_sequence_list or None, - max_sequences=max_sequences, - suppress_logs=suppress_logs, - console=console, - error_collector=error_collector, - organization=organization, - risk_score="extreme", - ) - except Exception as e: - error_collector.add_error(f"Alert API data fetching failed: {e}") - step_manager.complete_step(False, f"Alert API data fetching failed: {e}") - error_collector.print_summary(console, "Alert API Data Fetching Errors") - sys.exit(1) - - records, split_stats = object_split.split_all_records(records) - console.print( - f"[blue]🔀 Object split: {split_stats['alert_api_sequences']} alert sequence(s) → " - f"{split_stats['objects']} object sequence(s) " - f"({split_stats['sibling_objects']} sibling(s), " - f"{split_stats['fallback_sequences']} fallback, " - f"{split_stats['cross_deduped_siblings']} cross-deduped, " - f"{split_stats['same_frame_merges']} same-frame merge(s))[/]" - ) - # Anomaly, not a routine stat: printed only when it fires, so a dropped - # verdict stays distinguishable from an alert API that sends no score. - if split_stats["dropped_temporal_scores"]: - console.print( - f"[yellow]⚠️ {split_stats['dropped_temporal_scores']} scored alert " - "sequence(s) had no identifiable primary object (no bbox-sourced box " - "in the imported window); their temporal model score was dropped " - "rather than attributed to an arbitrary object[/]" - ) - - # Boxless alerts import as zero-object lanes the classify page cannot - # act on (#333); they are auto-skipped after annotation creation below. - boxless_alert_ids = sorted(shared.boxless_platform_alert_ids(records)) - - if not records and not args.dry_run: - step_manager.complete_step(False, "No records fetched from alert API") - sys.exit(0) - - # Post to annotation API (if not dry run) - if not args.dry_run: - console.print( - f"[blue]🚀 Posting {len(records)} records to annotation API...[/]" - ) - - try: - result = shared.post_records_to_annotation_api( - args.annotation_api_url, - records, - max_workers=worker_config.api_posting, - max_detection_workers=worker_config.detection_per_sequence, - suppress_logs=suppress_logs, - source_api=source_api, - force_url=(args.image_transfer == "url"), - ) - - # Capture import statistics in main stats and get successfully imported sequence IDs - stats["records_fetched"] = len(records) - stats["sequences_attempted_import"] = result["total_sequences"] - stats["sequences_import_successful"] = result["successful_sequences"] - stats["sequences_import_failed"] = result["failed_sequences"] - stats["detections_attempted_import"] = result["total_detections"] - stats["detections_import_successful"] = result["successful_detections"] - stats["detections_import_failed"] = result["failed_detections"] - stats["sequences_skipped"] = result.get("skipped_sequences", 0) - stats["sequences_refreshed"] = result.get("refreshed_sequences", 0) - stats["refresh_failures"] = result.get("refresh_failures", 0) - stats["refresh_skipped"] = result.get("refresh_skipped", 0) - if stats["refresh_failures"]: - # Feed the exit code and the ❌ summary: a backfill whose - # refreshes all failed must not report success. - error_collector.add_error( - f"{stats['refresh_failures']} temporal score refresh(es) failed" - ) - stats["detections_skipped"] = result.get("skipped_detections", 0) - successfully_imported_sequence_ids = result["successful_sequence_ids"] - - # Prepare step completion stats for display - step_stats = { - "Records fetched": len(records), - "Sequences posted": f"{result['successful_sequences']}/{result['total_sequences']}", - "Sequences skipped": result.get("skipped_sequences", 0), - "Detections skipped": result.get("skipped_detections", 0), - "Detections posted": f"{result['successful_detections']}/{result['total_detections']}", - } - - step_success = ( - result["failed_sequences"] == 0 and result["failed_detections"] == 0 - ) - step_message = ( - "Alert API data successfully imported" - if step_success - else "Alert API data imported with some failures" - ) - - step_manager.complete_step(step_success, step_message, step_stats) - - if result["failed_sequences"] > 0 or result["failed_detections"] > 0: - error_collector.add_warning( - f"{result['failed_sequences']} sequences and {result['failed_detections']} detections failed to import. " - "Enable --loglevel debug to see per-sequence errors." - ) - - except Exception as e: - error_collector.add_error(f"Failed to post data to annotation API: {e}") - step_manager.complete_step( - False, f"Failed to post data to annotation API: {e}" - ) - error_collector.print_summary(console, "Alert API Data Import Errors") - sys.exit(1) - else: - # For dry run, capture what would have been imported but don't set sequence IDs - stats["records_fetched"] = len(records) - step_stats = {"Records that would be posted": len(records)} - step_manager.complete_step( - True, "DRY RUN: Alert API data fetch completed", step_stats - ) - - # Step 2: Prepare sequences for annotation generation - step_manager.start_step( - 2, - "Sequence Preparation", - f"Preparing successfully imported {organization} sequences for annotation generation", - ) - - # Use only successfully imported sequences for annotation processing - sequence_ids = successfully_imported_sequence_ids - - if not sequence_ids: - step_message = "No sequences successfully imported - nothing to process for annotation generation" - step_manager.complete_step(True, step_message) - - # Boxless alerts from a previous run over this range may still - # need parking (an earlier skip failed, or the range predates the - # auto-skip feature); their lanes already exist, so skip works. - if boxless_alert_ids and not args.dry_run: - auto_skip_boxless( - args.annotation_api_url, - target_login, - target_password, - source_api, - boxless_alert_ids, - console, - error_collector, - ) - - # Show final summary with zero processing and exit gracefully. - # A pure backfill lands here: every sequence already existed, so - # nothing was "imported". This path exits before the detailed - # summary below, so the refresh counts must be reported here or - # they are invisible in exactly the run that produced them. - refresh_note = "" - title = f"⚠️ Processing Complete - {organization} - No Annotations Generated" - if ( - stats["sequences_refreshed"] - or stats["refresh_failures"] - or stats["refresh_skipped"] - ): - refresh_note = ( - f"\n\n[green]Temporal scores refreshed: " - f"{stats['sequences_refreshed']}[/]" - ) - if stats["refresh_skipped"]: - refresh_note += ( - f"\n[yellow]Skipped (score not determinable this run, " - f"existing values left intact): " - f"{stats['refresh_skipped']}[/]" - ) - if stats["refresh_failures"]: - refresh_note += ( - f"\n[red]Refresh failures: {stats['refresh_failures']}[/]" - ) - # Only a run that actually refreshed something may claim success; - # 0 refreshed with N failures is a failed backfill, not a green one. - if stats["refresh_failures"]: - title = ( - f"❌ Processing Complete - {organization} - " - f"{stats['refresh_failures']} Refresh Failure(s)" - ) - elif stats["sequences_refreshed"]: - title = ( - f"✅ Processing Complete - {organization} - " - f"{stats['sequences_refreshed']} Temporal Score(s) Refreshed" - ) - console.print() - panel = Panel( - f"[yellow]No sequences were successfully imported from {organization} alert API data.\n" - f"Check import statistics above for details (all sequences may already be imported — see Skipped).[/]" - + refresh_note, - title=title, - border_style="yellow", - padding=(1, 2), - ) - console.print(panel) - sys.exit(1 if stats["refresh_failures"] else 0) - - stats["total_sequences_for_annotation"] = len(sequence_ids) - step_stats = {"Successfully imported sequences": len(sequence_ids)} - step_manager.complete_step( - True, - f"Prepared {len(sequence_ids)} sequences for annotation generation", - step_stats, - ) - - # Step 3: Create sequence annotations with auto-generation - step_manager.start_step( - 3, - "Sequence Annotation Creation", - f"Creating sequence annotations for {len(sequence_ids)} sequences (auto-generation enabled)", - ) - - alert_api_seq_results = [] - annotation_auth_token = None - if not args.dry_run: - alert_api_seq_results = [ - r for r in result.get("sequence_results", []) if not r.get("skipped") - ] - # Captured here because the collection loop below rebinds `result`. - annotation_auth_token = result["auth_token"] - - with concurrent.futures.ThreadPoolExecutor( - max_workers=worker_config.annotation_processing - ) as executor: - # Submit all sequence annotation tasks - future_to_sequence_id = { - executor.submit( - annotate_split_sequence, - seq_result=seq_result, - annotation_api_url=args.annotation_api_url, - auth_token=annotation_auth_token, - dry_run=args.dry_run, - ): seq_result["sequence_id"] - for seq_result in alert_api_seq_results - } - - # Collect results with progress tracking - with LogSuppressor(suppress=suppress_logs): - with Progress( - SpinnerColumn(), - TextColumn("[bold blue]Creating sequence annotations"), - BarColumn(bar_width=40), - TaskProgressColumn(), - console=Console(), - transient=True, - ) as progress_bar: - task = progress_bar.add_task( - "Processing sequences", total=len(future_to_sequence_id) - ) - for future in concurrent.futures.as_completed( - future_to_sequence_id - ): - sequence_id = future_to_sequence_id[future] - try: - result = future.result() - - # Update annotation statistics - if result["errors"]: - stats["annotations_failed"] += 1 - for error in result["errors"]: - error_collector.add_error( - f"Sequence {sequence_id}: {error}" - ) - if "rolled back" in error: - stats["sequences_rolled_back"] += 1 - else: - stats["annotations_successful"] += 1 - - if result["annotation_created"]: - stats["annotations_created"] += 1 - - # Log progress (suppressed unless debug) - logger.debug( - f"Sequence {sequence_id}: " - f"annotation={'✓' if result['annotation_created'] else '✗'}, " - f"stage={result['final_stage'] or 'failed'}" - ) - progress_bar.advance(task) - - except Exception as e: - error_msg = f"Unexpected error processing sequence {sequence_id}: {e}" - error_collector.add_error(error_msg) - stats["annotations_failed"] += 1 - progress_bar.advance(task) - - # Complete Step 3 with annotation statistics - step_3_success = stats["annotations_failed"] == 0 - final_stats = { - "Sequences processed": stats["total_sequences_for_annotation"], - "Annotations successful": stats["annotations_successful"], - "Annotations failed": stats["annotations_failed"], - "Annotations created": stats["annotations_created"], - } - - step_3_message = ( - "All sequence annotations created successfully" - if step_3_success - else f"{stats['annotations_failed']} annotation(s) failed" - ) - if args.dry_run: - step_3_message = "DRY RUN: " + step_3_message - - step_manager.complete_step(step_3_success, step_3_message, final_stats) - - # Auto-skip boxless alerts (#333): their lanes exist now (sequence + - # annotation) but have zero objects, so park them via the skip overlay - # instead of leaving dead lanes in the classify queue. - skip_counts = {"skipped": 0, "already_skipped": 0, "failed": 0} - if boxless_alert_ids and not args.dry_run: - skip_counts = auto_skip_boxless( - args.annotation_api_url, - target_login, - target_password, - source_api, - boxless_alert_ids, - console, - error_collector, - ) - - # Show any accumulated errors/warnings - if error_collector.has_issues(): - error_collector.print_summary(console, "Processing Summary") - - # Enhanced final summary panel with import and annotation breakdown - console.print() - - # Determine overall success (critical failures, not including expected duplicates) - has_critical_failures = ( - stats["annotations_failed"] > 0 or error_collector.get_error_count() > 0 - ) - - success = not has_critical_failures - style = "green" if success else "red" - icon = "✅" if success else "❌" - - # Build comprehensive summary - summary_parts = [] - - # Alert API Import Section - if not args.dry_run: - import_section = f"""[bold cyan]ALERT API IMPORT:[/] -• Records fetched: {stats['records_fetched']} -• Sequences attempted: {stats['sequences_attempted_import']} -• Successfully imported: {stats['sequences_import_successful']} -• Skipped (already imported): {stats['sequences_skipped']} sequences / {stats['detections_skipped']} detections -• Failed: {stats['sequences_import_failed']}""" - if stats["sequences_refreshed"] or stats["refresh_failures"]: - import_section += ( - f"\n• Temporal scores refreshed: {stats['sequences_refreshed']}" - ) - if stats["refresh_failures"]: - import_section += ( - f"\n• [yellow]Refresh failures: {stats['refresh_failures']}[/]" - ) - if stats["sequences_rolled_back"] > 0: - import_section += f"\n• Rolled back: {stats['sequences_rolled_back']}" - summary_parts.append(import_section) - - # Annotation Generation Section - annotation_section = f"""[bold blue]ANNOTATION GENERATION:[/] -• Sequences processed: {stats['total_sequences_for_annotation']} -• Annotations successful: {stats['annotations_successful']} -• Annotations failed: {stats['annotations_failed']} -• Annotations created: {stats['annotations_created']}""" - if boxless_alert_ids: - annotation_section += ( - f"\n• Boxless alerts auto-skipped: {skip_counts['skipped']} " - f"(+{skip_counts['already_skipped']} already skipped, " - f"{skip_counts['failed']} failed): {boxless_alert_ids}" - ) - summary_parts.append(annotation_section) - - # Join sections - summary_text = "\n\n".join(summary_parts) - - # Add dry run notice - if args.dry_run: - summary_text += "\n\n[yellow]DRY RUN: No actual changes were made[/]" - - panel = Panel( - summary_text, - title=f"{icon} Processing Complete - {organization}", - border_style=style, - padding=(1, 2), - ) - console.print(panel) + if not shared.validate_available_env_variables(): + console.print("[red]❌ Missing required environment variables for alert API[/]") + sys.exit(1) - # Exit with appropriate code (only exit with error for critical failures) - if has_critical_failures: - sys.exit(1) - else: - sys.exit(0) + config = ImportConfig( + alert_api_url=args.alert_api_url, + login=shared.getenv_with_fallback("ALERT_API_LOGIN") or "", + password=shared.getenv_with_fallback("ALERT_API_PASSWORD") or "", + admin_login=shared.getenv_with_fallback("ALERT_API_ADMIN_LOGIN") or "", + admin_password=shared.getenv_with_fallback("ALERT_API_ADMIN_PASSWORD") or "", + annotation_api_url=args.annotation_api_url, + annotation_api_token=annotation_api_token, + date_from=args.date_from, + date_end=args.date_end, + source_api=source_api, + image_transfer=args.image_transfer, + max_workers=args.max_workers, + frames_limit=args.frames_limit, + max_sequences=max_sequences, + dry_run=args.dry_run, + # The CLI is the interactive caller: it wants the rich progress output + # that the worker suppresses. + quiet=False, + selected_sequence_ids=selected_sequence_list or None, + ) - except KeyboardInterrupt: - console.print("\n[yellow]⚠️ Processing interrupted by user[/]") - error_collector.print_summary(console, "Errors Before Interruption") - sys.exit(1) - except Exception as e: - error_collector.add_error(f"Unexpected error during processing: {e}") - console.print(f"\n[red]❌ Unexpected error during processing: {e}[/]") - error_collector.print_summary(console, "Critical Processing Errors") - sys.exit(1) + result = run_import(config) + sys.exit(0 if result.ok else 1) if __name__ == "__main__": diff --git a/annotation_api/scripts/data_transfer/ingestion/alert_api/runner.py b/annotation_api/scripts/data_transfer/ingestion/alert_api/runner.py new file mode 100644 index 00000000..fdbf0f7d --- /dev/null +++ b/annotation_api/scripts/data_transfer/ingestion/alert_api/runner.py @@ -0,0 +1,808 @@ +"""The importer as a library. + +`import.py` remains the CLI entry point (and keeps its name — the Makefile target +depends on it); it parses argv and environment into an ImportConfig and calls +run_import. The worker builds the same config from a connector row. One +implementation, two callers. +""" + +import concurrent.futures +import logging +import time +from dataclasses import dataclass, field +from datetime import date +from typing import Any, Dict, List, Optional, Set + +from rich.console import Console +from rich.panel import Panel +from rich.progress import ( + Progress, + SpinnerColumn, + TextColumn, + BarColumn, + TaskProgressColumn, +) + +from . import client as alert_api_client +from . import object_split +from . import shared +from .annotation_management import annotate_split_sequence +from .progress_management import ErrorCollector, StepManager, LogSuppressor +from .sequence_fetching import ( + fetch_detections_for_sequences, + filter_sequences, + list_sequences_within, + load_alert_api_metadata, +) +from .worker_config import WorkerConfig + +logger = logging.getLogger(__name__) + +DEFAULT_RISK_SCORE = "extreme" + + +@dataclass(frozen=True) +class ImportConfig: + alert_api_url: str + login: str + password: str + # The importer authenticates twice: a listing token, and an admin token for + # camera/organization metadata. A connector supplies one credential pair for + # both — the premise the verify endpoint tests. + admin_login: str + admin_password: str + annotation_api_url: str + annotation_api_token: str + date_from: date + date_end: date + source_api: str + image_transfer: Optional[str] = None + max_workers: int = 4 + frames_limit: int = 30 + max_sequences: int = 0 # 0 = unlimited + dry_run: bool = False + quiet: bool = True + organization_ids: Optional[Set[int]] = None + skip_platform_alert_ids: frozenset = field(default_factory=frozenset) + # Neutralizes the alert API's per-camera FWI filter so low-risk sequences are + # not silently dropped. Carried over from the CLI's behaviour. + risk_score: str = DEFAULT_RISK_SCORE + # CLI-only: `--sequence-list` restricts the run to these alert_api_id. + selected_sequence_ids: Optional[List[int]] = None + + +@dataclass +class OrganizationStats: + alerts_fetched: int = 0 + alerts_imported: int = 0 + # Both causes of "we did not import this, and that is fine": dropped by the + # pre-fetch filter as already present, AND reported by the annotation API as + # already existing at POST time. The two populations are disjoint. + alerts_skipped: int = 0 + alerts_failed: int = 0 + lanes_created: int = 0 + + +@dataclass +class ImportResult: + per_organization: dict[int, OrganizationStats] = field(default_factory=dict) + error: Optional[str] = None + + @property + def ok(self) -> bool: + return self.error is None + + +def _debug_logging_enabled() -> bool: + """True when the caller configured DEBUG logging. + + The CLI used to read `args.loglevel == "debug"` directly; it calls + `logging.basicConfig(level=...)` from the same flag, so reading the root + logger back keeps the behaviour without adding a config field. + """ + return logging.getLogger().isEnabledFor(logging.DEBUG) + + +def auto_skip_boxless( + annotation_api_url: str, + auth_token: str, + source_api: str, + boxless_alert_ids: List[int], + console: Console, + error_collector: ErrorCollector, +) -> dict: + """ + Best-effort auto-skip of boxless alerts (#333): park their zero-object + lanes via the skip overlay. Never raises — a skip failure must not fail + an otherwise successful import. + + Takes the already-resolved annotation-API token (`ImportConfig.annotation_api_token`) + rather than minting one from login/password: the worker self-mints a JWT and + has no ANNOTATOR_LOGIN/ANNOTATOR_PASSWORD in its environment, so a + get_auth_token call here would fail in that caller. + """ + counts = {"skipped": 0, "already_skipped": 0, "failed": 0} + try: + counts = shared.skip_boxless_alerts( + annotation_api_url, auth_token, source_api, boxless_alert_ids + ) + except Exception as exc: + counts["failed"] = len(boxless_alert_ids) + logging.warning("boxless auto-skip aborted: %s", exc) + console.print( + f"[blue]⏭️ Auto-skipped {counts['skipped']} boxless alert(s) " + f"({counts['already_skipped']} already skipped, " + f"{counts['failed']} failed): {boxless_alert_ids}[/]" + ) + if counts["failed"] > 0: + error_collector.add_warning( + f"{counts['failed']} boxless alert(s) could not be auto-skipped; " + "their zero-object lanes remain in the queue." + ) + return counts + + +def run_import(config: ImportConfig) -> ImportResult: + """Run the full alert-API import pipeline. + + This is `import.py:main()`'s pipeline, minus argv/environment handling: + authenticate, list the sequences for the date range, drop the ones we must + not or need not fetch, fetch their detections, object-split them, post them, + then write one annotation per posted object sequence. + + Returns an `ImportResult` instead of calling `sys.exit`; `error` is set for + every condition the CLI used to exit non-zero on. + """ + console = Console(quiet=config.quiet) + worker_config = WorkerConfig(config.max_workers) + suppress_logs = not _debug_logging_enabled() + step_manager = StepManager(console, show_timing=True) + error_collector = ErrorCollector() + + # Initialize comprehensive statistics + stats = { + # Import statistics (Step 1) + "records_fetched": 0, + "sequences_attempted_import": 0, + "sequences_import_successful": 0, + "sequences_import_failed": 0, + "sequences_skipped": 0, + "sequences_refreshed": 0, + "refresh_failures": 0, + "refresh_skipped": 0, + "detections_skipped": 0, + "detections_attempted_import": 0, + "detections_import_successful": 0, + "detections_import_failed": 0, + # Annotation statistics (Step 4) + "total_sequences_for_annotation": 0, + "annotations_successful": 0, + "annotations_failed": 0, + "annotations_created": 0, + "sequences_rolled_back": 0, + } + org_stats: Dict[int, OrganizationStats] = {} + + def stats_for(organization_id: int) -> OrganizationStats: + return org_stats.setdefault(organization_id, OrganizationStats()) + + # Label used in the console output; the CLI used to read it from + # ALERT_API_LOGIN, which is exactly what lands in `config.login`. + organization = config.login or "unknown" + + # Print header + console.print() + console.print( + Panel( + "[bold blue]Alert API Data Import & Processing[/]", + title="🔥 Pyronear Data Import", + border_style="blue", + padding=(0, 2), + ) + ) + + if _debug_logging_enabled(): + console.print( + f"[blue]ℹ️ Date range: {config.date_from} to {config.date_end}[/]" + ) + console.print( + f"[blue]ℹ️ Alert API: {config.alert_api_url} " + f"(source_api: {config.source_api})[/]" + ) + console.print(f"[blue]ℹ️ Worker config: {worker_config}[/]") + + try: + # Step 1: Fetch alert API data + successfully_imported_sequence_ids = [] + step_manager.start_step( + 1, + "Alert API Data Import", + f"Fetching {organization} data from {config.date_from} to {config.date_end} using {worker_config.base_workers} workers", + ) + + if not all( + [ + config.login, + config.password, + config.admin_login, + config.admin_password, + ] + ): + error_collector.add_error("Missing alert API credentials") + step_manager.complete_step(False, "Missing alert API credentials") + return ImportResult( + per_organization=org_stats, error="Missing alert API credentials" + ) + + # Get access tokens with progress display + auth_start_time = time.time() + with console.status( + f"[bold blue]🔐 Authenticating with alert API ({organization})...", + spinner="dots", + ) as status: + try: + status.update(f"[bold blue]🔐 Getting {organization} access token...") + access_token = alert_api_client.get_api_access_token( + api_endpoint=config.alert_api_url, + username=config.login, + password=config.password, + ) + + status.update("[bold blue]🔐 Getting admin access token...") + access_token_admin = alert_api_client.get_api_access_token( + api_endpoint=config.alert_api_url, + username=config.admin_login, + password=config.admin_password, + ) + + auth_duration = time.time() - auth_start_time + console.print( + f"[green]✅ Authentication successful[/] [dim]({auth_duration:.1f}s)[/]" + ) + + except Exception as e: + error_collector.add_error(f"Authentication failed: {e}") + step_manager.complete_step(False, f"Authentication failed: {e}") + return ImportResult( + per_organization=org_stats, error=f"Authentication failed: {e}" + ) + + # Fetch alert API records + try: + indexed_cameras, indexed_organizations = load_alert_api_metadata( + api_endpoint=config.alert_api_url, + access_token=access_token, + access_token_admin=access_token_admin, + console=console, + error_collector=error_collector, + ) + camera_org: Dict[int, Optional[int]] = { + camera_id: camera.get("organization_id") + for camera_id, camera in indexed_cameras.items() + } + + listed = list_sequences_within( + date_from=config.date_from, + date_end=config.date_end, + api_endpoint=config.alert_api_url, + access_token=access_token, + selected_sequence_list=config.selected_sequence_ids or None, + max_sequences=config.max_sequences, + suppress_logs=suppress_logs, + console=console, + risk_score=config.risk_score, + ) + + skip_ids = set(config.skip_platform_alert_ids) + for sequence in listed: + org_id = camera_org.get(sequence.get("camera_id")) + if org_id is not None: + stats_for(org_id).alerts_fetched += 1 + if sequence["id"] in skip_ids: + stats_for(org_id).alerts_skipped += 1 + + # Applied BEFORE the per-sequence detection fetch below: a re-run of + # an already-imported day then costs one listing call and zero + # detection calls. + sequences = filter_sequences( + listed, + camera_org=camera_org, + organization_ids=config.organization_ids, + skip_ids=skip_ids, + ) + if config.organization_ids is not None or skip_ids: + console.print( + f"[blue]🔍 Filtered sequences before detection fetch[/] " + f"[dim]({len(listed) - len(sequences)} skipped, " + f"{len(sequences)} remaining)[/]" + ) + + records = fetch_detections_for_sequences( + sequences=sequences, + indexed_cameras=indexed_cameras, + indexed_organizations=indexed_organizations, + api_endpoint=config.alert_api_url, + access_token=access_token, + detections_limit=config.frames_limit, + detections_order_by="asc", + worker_config=worker_config, + suppress_logs=suppress_logs, + console=console, + error_collector=error_collector, + organization=organization, + ) + except Exception as e: + error_collector.add_error(f"Alert API data fetching failed: {e}") + step_manager.complete_step(False, f"Alert API data fetching failed: {e}") + error_collector.print_summary(console, "Alert API Data Fetching Errors") + return ImportResult( + per_organization=org_stats, + error=f"Alert API data fetching failed: {e}", + ) + + records, split_stats = object_split.split_all_records(records) + console.print( + f"[blue]🔀 Object split: {split_stats['alert_api_sequences']} alert sequence(s) → " + f"{split_stats['objects']} object sequence(s) " + f"({split_stats['sibling_objects']} sibling(s), " + f"{split_stats['fallback_sequences']} fallback, " + f"{split_stats['cross_deduped_siblings']} cross-deduped, " + f"{split_stats['same_frame_merges']} same-frame merge(s))[/]" + ) + # Anomaly, not a routine stat: printed only when it fires, so a dropped + # verdict stays distinguishable from an alert API that sends no score. + if split_stats["dropped_temporal_scores"]: + console.print( + f"[yellow]⚠️ {split_stats['dropped_temporal_scores']} scored alert " + "sequence(s) had no identifiable primary object (no bbox-sourced box " + "in the imported window); their temporal model score was dropped " + "rather than attributed to an arbitrary object[/]" + ) + + # Boxless alerts import as zero-object lanes the classify page cannot + # act on (#333); they are auto-skipped after annotation creation below. + boxless_alert_ids = sorted(shared.boxless_platform_alert_ids(records)) + + if not records and not config.dry_run: + step_manager.complete_step(False, "No records fetched from alert API") + return ImportResult(per_organization=org_stats) + + # Post to annotation API (if not dry run) + if not config.dry_run: + console.print( + f"[blue]🚀 Posting {len(records)} records to annotation API...[/]" + ) + + try: + result = shared.post_records_to_annotation_api( + config.annotation_api_url, + records, + max_workers=worker_config.api_posting, + max_detection_workers=worker_config.detection_per_sequence, + suppress_logs=suppress_logs, + source_api=config.source_api, + force_url=(config.image_transfer == "url"), + auth_token=config.annotation_api_token, + ) + + # Capture import statistics in main stats and get successfully imported sequence IDs + stats["records_fetched"] = len(records) + stats["sequences_attempted_import"] = result["total_sequences"] + stats["sequences_import_successful"] = result["successful_sequences"] + stats["sequences_import_failed"] = result["failed_sequences"] + stats["detections_attempted_import"] = result["total_detections"] + stats["detections_import_successful"] = result["successful_detections"] + stats["detections_import_failed"] = result["failed_detections"] + stats["sequences_skipped"] = result.get("skipped_sequences", 0) + stats["sequences_refreshed"] = result.get("refreshed_sequences", 0) + stats["refresh_failures"] = result.get("refresh_failures", 0) + stats["refresh_skipped"] = result.get("refresh_skipped", 0) + if stats["refresh_failures"]: + # Feed the result status and the ❌ summary: a backfill whose + # refreshes all failed must not report success. + error_collector.add_error( + f"{stats['refresh_failures']} temporal score refresh(es) failed" + ) + stats["detections_skipped"] = result.get("skipped_detections", 0) + successfully_imported_sequence_ids = result["successful_sequence_ids"] + + # Prepare step completion stats for display + step_stats = { + "Records fetched": len(records), + "Sequences posted": f"{result['successful_sequences']}/{result['total_sequences']}", + "Sequences skipped": result.get("skipped_sequences", 0), + "Detections skipped": result.get("skipped_detections", 0), + "Detections posted": f"{result['successful_detections']}/{result['total_detections']}", + } + + step_success = ( + result["failed_sequences"] == 0 and result["failed_detections"] == 0 + ) + step_message = ( + "Alert API data successfully imported" + if step_success + else "Alert API data imported with some failures" + ) + + step_manager.complete_step(step_success, step_message, step_stats) + + if result["failed_sequences"] > 0 or result["failed_detections"] > 0: + error_collector.add_warning( + f"{result['failed_sequences']} sequences and {result['failed_detections']} detections failed to import. " + "Enable --loglevel debug to see per-sequence errors." + ) + + except Exception as e: + error_collector.add_error(f"Failed to post data to annotation API: {e}") + step_manager.complete_step( + False, f"Failed to post data to annotation API: {e}" + ) + error_collector.print_summary(console, "Alert API Data Import Errors") + return ImportResult( + per_organization=org_stats, + error=f"Failed to post data to annotation API: {e}", + ) + + # Bookkeeping only, and deliberately outside the try above: a bug in + # here must never be reported as "failed to post" on a run whose + # transfer actually succeeded. + _accumulate_post_stats(records, result, org_stats) + else: + # For dry run, capture what would have been imported but don't set sequence IDs + stats["records_fetched"] = len(records) + step_stats = {"Records that would be posted": len(records)} + step_manager.complete_step( + True, "DRY RUN: Alert API data fetch completed", step_stats + ) + + # Step 2: Prepare sequences for annotation generation + step_manager.start_step( + 2, + "Sequence Preparation", + f"Preparing successfully imported {organization} sequences for annotation generation", + ) + + # Use only successfully imported sequences for annotation processing + sequence_ids = successfully_imported_sequence_ids + + if not sequence_ids: + step_message = "No sequences successfully imported - nothing to process for annotation generation" + step_manager.complete_step(True, step_message) + + # Boxless alerts from a previous run over this range may still + # need parking (an earlier skip failed, or the range predates the + # auto-skip feature); their lanes already exist, so skip works. + if boxless_alert_ids and not config.dry_run: + auto_skip_boxless( + config.annotation_api_url, + config.annotation_api_token, + config.source_api, + boxless_alert_ids, + console, + error_collector, + ) + + # Show final summary with zero processing and exit gracefully. + # A pure backfill lands here: every sequence already existed, so + # nothing was "imported". This path returns before the detailed + # summary below, so the refresh counts must be reported here or + # they are invisible in exactly the run that produced them. + refresh_note = "" + title = f"⚠️ Processing Complete - {organization} - No Annotations Generated" + if ( + stats["sequences_refreshed"] + or stats["refresh_failures"] + or stats["refresh_skipped"] + ): + refresh_note = ( + f"\n\n[green]Temporal scores refreshed: " + f"{stats['sequences_refreshed']}[/]" + ) + if stats["refresh_skipped"]: + refresh_note += ( + f"\n[yellow]Skipped (score not determinable this run, " + f"existing values left intact): " + f"{stats['refresh_skipped']}[/]" + ) + if stats["refresh_failures"]: + refresh_note += ( + f"\n[red]Refresh failures: {stats['refresh_failures']}[/]" + ) + # Only a run that actually refreshed something may claim success; + # 0 refreshed with N failures is a failed backfill, not a green one. + if stats["refresh_failures"]: + title = ( + f"❌ Processing Complete - {organization} - " + f"{stats['refresh_failures']} Refresh Failure(s)" + ) + elif stats["sequences_refreshed"]: + title = ( + f"✅ Processing Complete - {organization} - " + f"{stats['sequences_refreshed']} Temporal Score(s) Refreshed" + ) + console.print() + panel = Panel( + f"[yellow]No sequences were successfully imported from {organization} alert API data.\n" + f"Check import statistics above for details (all sequences may already be imported — see Skipped).[/]" + + refresh_note, + title=title, + border_style="yellow", + padding=(1, 2), + ) + console.print(panel) + if stats["refresh_failures"]: + return ImportResult( + per_organization=org_stats, + error=( + f"{stats['refresh_failures']} temporal score refresh(es) failed" + ), + ) + return ImportResult(per_organization=org_stats) + + stats["total_sequences_for_annotation"] = len(sequence_ids) + step_stats = {"Successfully imported sequences": len(sequence_ids)} + step_manager.complete_step( + True, + f"Prepared {len(sequence_ids)} sequences for annotation generation", + step_stats, + ) + + # Step 3: Create sequence annotations with auto-generation + step_manager.start_step( + 3, + "Sequence Annotation Creation", + f"Creating sequence annotations for {len(sequence_ids)} sequences (auto-generation enabled)", + ) + + alert_api_seq_results = [] + if not config.dry_run: + alert_api_seq_results = [ + r for r in result.get("sequence_results", []) if not r.get("skipped") + ] + + with concurrent.futures.ThreadPoolExecutor( + max_workers=worker_config.annotation_processing + ) as executor: + # Submit all sequence annotation tasks + future_to_sequence_id = { + executor.submit( + annotate_split_sequence, + seq_result=seq_result, + annotation_api_url=config.annotation_api_url, + dry_run=config.dry_run, + auth_token=config.annotation_api_token, + ): seq_result["sequence_id"] + for seq_result in alert_api_seq_results + } + + # Collect results with progress tracking + with LogSuppressor(suppress=suppress_logs): + with Progress( + SpinnerColumn(), + TextColumn("[bold blue]Creating sequence annotations"), + BarColumn(bar_width=40), + TaskProgressColumn(), + console=Console(quiet=config.quiet), + transient=True, + ) as progress_bar: + task = progress_bar.add_task( + "Processing sequences", total=len(future_to_sequence_id) + ) + for future in concurrent.futures.as_completed( + future_to_sequence_id + ): + sequence_id = future_to_sequence_id[future] + try: + result = future.result() + + # Update annotation statistics + if result["errors"]: + stats["annotations_failed"] += 1 + for error in result["errors"]: + error_collector.add_error( + f"Sequence {sequence_id}: {error}" + ) + if "rolled back" in error: + stats["sequences_rolled_back"] += 1 + else: + stats["annotations_successful"] += 1 + + if result["annotation_created"]: + stats["annotations_created"] += 1 + + # Log progress (suppressed unless debug) + logger.debug( + f"Sequence {sequence_id}: " + f"annotation={'✓' if result['annotation_created'] else '✗'}, " + f"stage={result['final_stage'] or 'failed'}" + ) + progress_bar.advance(task) + + except Exception as e: + error_msg = f"Unexpected error processing sequence {sequence_id}: {e}" + error_collector.add_error(error_msg) + stats["annotations_failed"] += 1 + progress_bar.advance(task) + + # Complete Step 3 with annotation statistics + step_3_success = stats["annotations_failed"] == 0 + final_stats = { + "Sequences processed": stats["total_sequences_for_annotation"], + "Annotations successful": stats["annotations_successful"], + "Annotations failed": stats["annotations_failed"], + "Annotations created": stats["annotations_created"], + } + + step_3_message = ( + "All sequence annotations created successfully" + if step_3_success + else f"{stats['annotations_failed']} annotation(s) failed" + ) + if config.dry_run: + step_3_message = "DRY RUN: " + step_3_message + + step_manager.complete_step(step_3_success, step_3_message, final_stats) + + # Auto-skip boxless alerts (#333): their lanes exist now (sequence + + # annotation) but have zero objects, so park them via the skip overlay + # instead of leaving dead lanes in the classify queue. + skip_counts = {"skipped": 0, "already_skipped": 0, "failed": 0} + if boxless_alert_ids and not config.dry_run: + skip_counts = auto_skip_boxless( + config.annotation_api_url, + config.annotation_api_token, + config.source_api, + boxless_alert_ids, + console, + error_collector, + ) + + # Show any accumulated errors/warnings + if error_collector.has_issues(): + error_collector.print_summary(console, "Processing Summary") + + # Enhanced final summary panel with import and annotation breakdown + console.print() + + # Determine overall success (critical failures, not including expected duplicates) + has_critical_failures = ( + stats["annotations_failed"] > 0 or error_collector.get_error_count() > 0 + ) + + success = not has_critical_failures + style = "green" if success else "red" + icon = "✅" if success else "❌" + + # Build comprehensive summary + summary_parts = [] + + # Alert API Import Section + if not config.dry_run: + import_section = f"""[bold cyan]ALERT API IMPORT:[/] +• Records fetched: {stats['records_fetched']} +• Sequences attempted: {stats['sequences_attempted_import']} +• Successfully imported: {stats['sequences_import_successful']} +• Skipped (already imported): {stats['sequences_skipped']} sequences / {stats['detections_skipped']} detections +• Failed: {stats['sequences_import_failed']}""" + if stats["sequences_refreshed"] or stats["refresh_failures"]: + import_section += ( + f"\n• Temporal scores refreshed: {stats['sequences_refreshed']}" + ) + if stats["refresh_failures"]: + import_section += ( + f"\n• [yellow]Refresh failures: {stats['refresh_failures']}[/]" + ) + if stats["sequences_rolled_back"] > 0: + import_section += f"\n• Rolled back: {stats['sequences_rolled_back']}" + summary_parts.append(import_section) + + # Annotation Generation Section + annotation_section = f"""[bold blue]ANNOTATION GENERATION:[/] +• Sequences processed: {stats['total_sequences_for_annotation']} +• Annotations successful: {stats['annotations_successful']} +• Annotations failed: {stats['annotations_failed']} +• Annotations created: {stats['annotations_created']}""" + if boxless_alert_ids: + annotation_section += ( + f"\n• Boxless alerts auto-skipped: {skip_counts['skipped']} " + f"(+{skip_counts['already_skipped']} already skipped, " + f"{skip_counts['failed']} failed): {boxless_alert_ids}" + ) + summary_parts.append(annotation_section) + + # Join sections + summary_text = "\n\n".join(summary_parts) + + # Add dry run notice + if config.dry_run: + summary_text += "\n\n[yellow]DRY RUN: No actual changes were made[/]" + + panel = Panel( + summary_text, + title=f"{icon} Processing Complete - {organization}", + border_style=style, + padding=(1, 2), + ) + console.print(panel) + + if has_critical_failures: + return ImportResult( + per_organization=org_stats, + error=( + f"{stats['annotations_failed']} annotation(s) failed, " + f"{error_collector.get_error_count()} error(s) collected" + ), + ) + return ImportResult(per_organization=org_stats) + + except KeyboardInterrupt: + console.print("\n[yellow]⚠️ Processing interrupted by user[/]") + error_collector.print_summary(console, "Errors Before Interruption") + return ImportResult( + per_organization=org_stats, error="Processing interrupted by user" + ) + except Exception as e: + error_collector.add_error(f"Unexpected error during processing: {e}") + console.print(f"\n[red]❌ Unexpected error during processing: {e}[/]") + error_collector.print_summary(console, "Critical Processing Errors") + return ImportResult( + per_organization=org_stats, error=f"Unexpected error during processing: {e}" + ) + + +def _accumulate_post_stats( + records: List[Dict[str, Any]], + post_result: Dict[str, Any], + org_stats: Dict[int, OrganizationStats], +) -> None: + """Attribute one posting run's outcome to the organizations it touched. + + Object-splitting turns one alert into several "lanes" (one annotation + sequence per detected object), so lanes are counted directly while the + alert-level counters roll their lanes up: an alert counts as failed if any + of its lanes failed, imported if any lane was created, and skipped when the + annotation API reported every one of its lanes as already existing. + + `post_records_to_annotation_api` only records a `sequence_results` entry for + lanes that were created or skipped, so the failed lanes are the posted ones + it did not report back. + """ + lane_alert: Dict[int, int] = {} + lane_org: Dict[int, Optional[int]] = {} + for record in records: + lane_id = record["sequence_id"] + lane_alert[lane_id] = record.get("platform_alert_id", lane_id) + lane_org[lane_id] = record.get("organization_id") + + sequence_results = post_result.get("sequence_results", []) + created = { + r["alert_api_sequence_id"] for r in sequence_results if not r.get("skipped") + } + already_present = { + r["alert_api_sequence_id"] for r in sequence_results if r.get("skipped") + } + failed = set(lane_alert) - created - already_present + + lanes_by_alert: Dict[int, List[int]] = {} + for lane_id, alert_id in lane_alert.items(): + lanes_by_alert.setdefault(alert_id, []).append(lane_id) + + for alert_id, lane_ids in lanes_by_alert.items(): + org_id = next( + ( + lane_org[lane_id] + for lane_id in lane_ids + if lane_org[lane_id] is not None + ), + None, + ) + if org_id is None: + continue + entry = org_stats.setdefault(org_id, OrganizationStats()) + entry.lanes_created += sum(1 for lane_id in lane_ids if lane_id in created) + if any(lane_id in failed for lane_id in lane_ids): + entry.alerts_failed += 1 + elif any(lane_id in created for lane_id in lane_ids): + entry.alerts_imported += 1 + else: + entry.alerts_skipped += 1 diff --git a/annotation_api/scripts/data_transfer/ingestion/alert_api/sequence_fetching.py b/annotation_api/scripts/data_transfer/ingestion/alert_api/sequence_fetching.py index 2d2bb7b6..94a92da5 100644 --- a/annotation_api/scripts/data_transfer/ingestion/alert_api/sequence_fetching.py +++ b/annotation_api/scripts/data_transfer/ingestion/alert_api/sequence_fetching.py @@ -38,7 +38,7 @@ import logging import time from datetime import date, timedelta -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Set, Tuple from rich.console import Console from rich.progress import ( @@ -142,6 +142,35 @@ def fetch_sequences_for_date( return sequences +def filter_sequences( + sequences: List[Dict[str, Any]], + *, + camera_org: Dict[int, Optional[int]], + organization_ids: Optional[Set[int]], + skip_ids: Set[int], +) -> List[Dict[str, Any]]: + """Drop sequences we must not, or need not, fetch detections for. + + Applied immediately after the date listing and before any per-sequence + detection call — that ordering is the whole point. The importer otherwise + only discovers "already exists" at POST time, after paying for every fetch. + + A sequence whose camera is absent from the index cannot be attributed to an + organization; it is dropped when filtering by organization (importing it + would silently ingest an org the operator never enabled) and kept when not. + """ + kept = [] + for sequence in sequences: + if sequence["id"] in skip_ids: + continue + if organization_ids is not None: + org = camera_org.get(sequence.get("camera_id")) + if org is None or org not in organization_ids: + continue + kept.append(sequence) + return kept + + def process_single_sequence_detections( sequence: Dict[str, Any], indexed_cameras: Dict[int, Dict[str, Any]], @@ -308,6 +337,58 @@ def fetch_all_sequences_within( if error_collector is None: error_collector = ErrorCollector() + indexed_cameras, indexed_organizations = load_alert_api_metadata( + api_endpoint=api_endpoint, + access_token=access_token, + access_token_admin=access_token_admin, + console=console, + error_collector=error_collector, + ) + sequences = list_sequences_within( + date_from=date_from, + date_end=date_end, + api_endpoint=api_endpoint, + access_token=access_token, + selected_sequence_list=selected_sequence_list, + max_sequences=max_sequences, + suppress_logs=suppress_logs, + console=console, + risk_score=risk_score, + ) + return fetch_detections_for_sequences( + sequences=sequences, + indexed_cameras=indexed_cameras, + indexed_organizations=indexed_organizations, + api_endpoint=api_endpoint, + access_token=access_token, + detections_limit=detections_limit, + detections_order_by=detections_order_by, + worker_config=worker_config, + suppress_logs=suppress_logs, + console=console, + error_collector=error_collector, + organization=organization, + ) + + +def load_alert_api_metadata( + api_endpoint: str, + access_token: str, + access_token_admin: str, + console: Optional[Console] = None, + error_collector: Optional[ErrorCollector] = None, +) -> Tuple[Dict[int, Dict[str, Any]], Dict[int, Dict[str, Any]]]: + """Load the camera and organization indexes used to enrich records. + + Split out of `fetch_all_sequences_within` so a caller that needs the camera + index *before* deciding which sequences deserve a detection fetch (see + `filter_sequences`) can reuse it instead of listing cameras twice. + """ + if console is None: + console = Console() + if error_collector is None: + error_collector = ErrorCollector() + # Load metadata with progress display metadata_start_time = time.time() with console.status( @@ -340,6 +421,28 @@ def fetch_all_sequences_within( error_collector.add_error(error_msg) raise Exception(error_msg) + return indexed_cameras, indexed_organizations + + +def list_sequences_within( + date_from: date, + date_end: date, + api_endpoint: str, + access_token: str, + selected_sequence_list: Optional[List[int]] = None, + max_sequences: Optional[int] = None, + suppress_logs: bool = True, + console: Optional[Console] = None, + risk_score: Optional[str] = None, +) -> List[Dict[str, Any]]: + """List the alert sequences in the date range, without their detections. + + Split out of `fetch_all_sequences_within` so the organization / already-seen + filters can run between the listing and the per-sequence detection fetch. + """ + if console is None: + console = Console() + # Prepare date range dates = get_dates_within(date_from=date_from, date_end=date_end) @@ -414,6 +517,33 @@ def fetch_all_sequences_within( console.print(f"[green]✅ Found {len(sequences)} sequences[/]") + return sequences + + +def fetch_detections_for_sequences( + sequences: List[Dict[str, Any]], + indexed_cameras: Dict[int, Dict[str, Any]], + indexed_organizations: Dict[int, Dict[str, Any]], + api_endpoint: str, + access_token: str, + detections_limit: int, + detections_order_by: str, + worker_config: WorkerConfig, + suppress_logs: bool = True, + console: Optional[Console] = None, + error_collector: Optional[ErrorCollector] = None, + organization: Optional[str] = None, +) -> List[Dict[str, Any]]: + """Fetch each listed sequence's detections and flatten them into records. + + Split out of `fetch_all_sequences_within`; this is the expensive stage the + `filter_sequences` short-circuit exists to keep work out of. + """ + if console is None: + console = Console() + if error_collector is None: + error_collector = ErrorCollector() + # Without this the two cases are indistinguishable: an alert API that # predates temporal validation imports exactly like a day where nothing was # scored — every sequence NULL, run reports success. Warn rather than fail: diff --git a/annotation_api/scripts/data_transfer/ingestion/alert_api/shared.py b/annotation_api/scripts/data_transfer/ingestion/alert_api/shared.py index 25f10831..0865e3e1 100644 --- a/annotation_api/scripts/data_transfer/ingestion/alert_api/shared.py +++ b/annotation_api/scripts/data_transfer/ingestion/alert_api/shared.py @@ -765,6 +765,7 @@ def post_records_to_annotation_api( suppress_logs: bool = True, source_api: str = "pyronear_french", force_url: bool = False, + auth_token: Optional[str] = None, ) -> Dict: """ Post multiple sequences and their detections to the annotation API. @@ -776,6 +777,10 @@ def post_records_to_annotation_api( max_detection_workers: Maximum number of workers for detection creation within each sequence suppress_logs: Whether to suppress log output during progress display source_api: Source API enum value (pyronear_french, api_cenia, etc.) + auth_token: Annotation API token to post with. When None, credentials are + read from the environment and exchanged for a token — the worker + supplies its own self-minted token instead, so that no plaintext + annotation-API password has to exist in its environment. Returns: Dictionary with summary statistics including list of successfully imported sequence IDs @@ -801,8 +806,11 @@ def post_records_to_annotation_api( } # Resolve credentials and get a single auth token up-front to avoid repeated logins - login, password = get_annotation_credentials(annotation_api_url) - auth_token = get_auth_token(annotation_api_url, username=login, password=password) + if auth_token is None: + login, password = get_annotation_credentials(annotation_api_url) + auth_token = get_auth_token( + annotation_api_url, username=login, password=password + ) # Group records by sequence grouped_records = group_records_by_sequence(records) diff --git a/annotation_api/src/app/api/api_v1/endpoints/connectors.py b/annotation_api/src/app/api/api_v1/endpoints/connectors.py new file mode 100644 index 00000000..15cc6560 --- /dev/null +++ b/annotation_api/src/app/api/api_v1/endpoints/connectors.py @@ -0,0 +1,280 @@ +# Copyright (C) 2025, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +from datetime import UTC, date, datetime, timedelta + +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError +from sqlmodel.ext.asyncio.session import AsyncSession + +from app.auth.dependencies import get_current_superuser +from app.db import get_session +from app.models import ( + AlertApiConnector, + AlertApiConnectorOrganization, + AlertApiImportCoverage, + User, +) +from app.schemas.connector import ( + ConnectorCreate, + ConnectorOrganizationRead, + ConnectorOrganizationUpdate, + ConnectorRead, + ConnectorTestRequest, + ConnectorTestResult, + ConnectorUpdate, + CoverageCellRead, + VerifyResult, +) +from app.services.connector_verify import check_connector_credentials, verify_connector +from app.services.secrets import SecretKeyMissingError, encrypt_secret + +router = APIRouter() + + +async def _to_read( + session: AsyncSession, connector: AlertApiConnector +) -> ConnectorRead: + counts = ( + await session.execute( + select( + func.count(AlertApiConnectorOrganization.id), + func.count(AlertApiConnectorOrganization.id).filter( + AlertApiConnectorOrganization.is_enabled.is_(True) + ), + ).where(AlertApiConnectorOrganization.connector_id == connector.id) + ) + ).one() + return ConnectorRead( + id=connector.id, + name=connector.name, + base_url=connector.base_url, + source_api=connector.source_api, + login=connector.login, + has_password=bool(connector.password_encrypted), + is_enabled=connector.is_enabled, + trailing_days=connector.trailing_days, + image_transfer=connector.image_transfer, + last_verified_at=connector.last_verified_at, + last_verify_error=connector.last_verify_error, + organizations_total=counts[0], + organizations_enabled=counts[1], + ) + + +async def _get_or_404(session: AsyncSession, connector_id: int) -> AlertApiConnector: + connector = await session.get(AlertApiConnector, connector_id) + if connector is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Connector not found" + ) + return connector + + +def _encrypt_or_400(password: str) -> str: + try: + return encrypt_secret(password) + except SecretKeyMissingError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) + + +@router.get("/", response_model=list[ConnectorRead]) +async def list_connectors( + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_superuser), +) -> list[ConnectorRead]: + """List every configured alert API connector.""" + connectors = ( + ( + await session.execute( + select(AlertApiConnector).order_by(AlertApiConnector.name) + ) + ) + .scalars() + .all() + ) + return [await _to_read(session, connector) for connector in connectors] + + +@router.post("/", response_model=ConnectorRead, status_code=status.HTTP_201_CREATED) +async def create_connector( + payload: ConnectorCreate, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_superuser), +) -> ConnectorRead: + """Register an alert API. The password is encrypted before it is stored and + is never returned.""" + connector = AlertApiConnector( + name=payload.name, + base_url=payload.base_url, + source_api=payload.source_api, + login=payload.login, + password_encrypted=_encrypt_or_400(payload.password), + is_enabled=payload.is_enabled, + trailing_days=payload.trailing_days, + image_transfer=payload.image_transfer, + ) + session.add(connector) + try: + await session.commit() + except IntegrityError: + await session.rollback() + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "A connector already exists for this base URL or source API. " + "Each source API may be claimed by only one connector." + ), + ) + await session.refresh(connector) + return await _to_read(session, connector) + + +@router.post("/test", response_model=ConnectorTestResult) +async def test_credentials( + payload: ConnectorTestRequest, + current_user: User = Depends(get_current_superuser), +) -> ConnectorTestResult: + """Stateless pre-save credential check: token exchange + organization + listing, nothing persisted. Catches both real failure modes — wrong + password, and an org-scoped credential ("Incompatible token scope.").""" + return await check_connector_credentials( + payload.base_url, payload.login, payload.password + ) + + +@router.patch("/{connector_id}", response_model=ConnectorRead) +async def update_connector( + connector_id: int, + payload: ConnectorUpdate, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_superuser), +) -> ConnectorRead: + """Update a connector. Omitting `password` leaves the stored one intact.""" + connector = await _get_or_404(session, connector_id) + fields = payload.model_dump(exclude_unset=True) + password = fields.pop("password", None) + if password is not None: + connector.password_encrypted = _encrypt_or_400(password) + for key, value in fields.items(): + setattr(connector, key, value) + session.add(connector) + try: + await session.commit() + except IntegrityError: + await session.rollback() + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="A connector already exists for this base URL.", + ) + await session.refresh(connector) + return await _to_read(session, connector) + + +@router.delete("/{connector_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_connector( + connector_id: int, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_superuser), +) -> None: + """Delete a connector, its discovered organizations, and its coverage.""" + connector = await _get_or_404(session, connector_id) + await session.delete(connector) + await session.commit() + + +@router.post("/{connector_id}/verify", response_model=VerifyResult) +async def verify( + connector_id: int, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_superuser), +) -> VerifyResult: + """Test the credential, discover organizations, and report how many of them + actually appear in a one-day sample listing.""" + connector = await _get_or_404(session, connector_id) + return await verify_connector(session, connector, today=datetime.now(UTC).date()) + + +@router.get( + "/{connector_id}/organizations", response_model=list[ConnectorOrganizationRead] +) +async def list_connector_organizations( + connector_id: int, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_superuser), +) -> list[AlertApiConnectorOrganization]: + """Organizations discovered on this connector, in name order.""" + await _get_or_404(session, connector_id) + result = await session.execute( + select(AlertApiConnectorOrganization) + .where(AlertApiConnectorOrganization.connector_id == connector_id) + .order_by(AlertApiConnectorOrganization.name) + ) + return list(result.scalars().all()) + + +@router.patch( + "/{connector_id}/organizations/{organization_id}", + response_model=ConnectorOrganizationRead, +) +async def update_connector_organization( + connector_id: int, + organization_id: int, + payload: ConnectorOrganizationUpdate, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_superuser), +) -> AlertApiConnectorOrganization: + """Include or exclude one organization from the daily import.""" + await _get_or_404(session, connector_id) + row = ( + await session.execute( + select(AlertApiConnectorOrganization).where( + AlertApiConnectorOrganization.connector_id == connector_id, + AlertApiConnectorOrganization.organization_id == organization_id, + ) + ) + ).scalar_one_or_none() + if row is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Organization not found on this connector; run verify first.", + ) + row.is_enabled = payload.is_enabled + # enabled_at marks when this organization FIRST entered ingestion. The + # heatmap greys out days before it, so it must never be moved once set. + if payload.is_enabled and row.enabled_at is None: + row.enabled_at = datetime.now(UTC) + session.add(row) + await session.commit() + await session.refresh(row) + return row + + +@router.get("/{connector_id}/coverage", response_model=list[CoverageCellRead]) +async def read_coverage( + connector_id: int, + date_from: date | None = None, + date_end: date | None = None, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_superuser), +) -> list[AlertApiImportCoverage]: + """Coverage cells for the heatmap. Defaults to the last 30 days.""" + await _get_or_404(session, connector_id) + today = datetime.now(UTC).date() + date_end = date_end or today + date_from = date_from or (date_end - timedelta(days=29)) + result = await session.execute( + select(AlertApiImportCoverage) + .where( + AlertApiImportCoverage.connector_id == connector_id, + AlertApiImportCoverage.covered_date >= date_from, + AlertApiImportCoverage.covered_date <= date_end, + ) + .order_by( + AlertApiImportCoverage.organization_id, AlertApiImportCoverage.covered_date + ) + ) + return list(result.scalars().all()) diff --git a/annotation_api/src/app/api/api_v1/endpoints/users.py b/annotation_api/src/app/api/api_v1/endpoints/users.py index 6b1503e5..22c7a442 100644 --- a/annotation_api/src/app/api/api_v1/endpoints/users.py +++ b/annotation_api/src/app/api/api_v1/endpoints/users.py @@ -186,6 +186,17 @@ async def update_user_password( """Update a user's password (admin only).""" user_crud = UserCRUD(session) + target_user = await user_crud.get_by_id(user_id) + if not target_user: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="User not found" + ) + if target_user.username == settings.WORKER_USERNAME: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Cannot modify the system worker user", + ) + user = await user_crud.update_user_password(user_id, password_update) if not user: raise HTTPException( diff --git a/annotation_api/src/app/api/api_v1/router.py b/annotation_api/src/app/api/api_v1/router.py index e10f4e90..11256455 100644 --- a/annotation_api/src/app/api/api_v1/router.py +++ b/annotation_api/src/app/api/api_v1/router.py @@ -8,6 +8,7 @@ from app.api.api_v1.endpoints import ( auto_annotate, cameras, + connectors, detection_annotations, detections, organizations, @@ -27,6 +28,7 @@ # User management endpoints api_router.include_router(users.router, prefix="/users", tags=["users"]) +api_router.include_router(connectors.router, prefix="/connectors", tags=["connectors"]) api_router.include_router(detections.router, prefix="/detections", tags=["detections"]) api_router.include_router( diff --git a/annotation_api/src/app/core/config.py b/annotation_api/src/app/core/config.py index 4556d81d..718a5b63 100644 --- a/annotation_api/src/app/core/config.py +++ b/annotation_api/src/app/core/config.py @@ -38,6 +38,18 @@ class Settings(BaseSettings): ANNOTATOR_LOGIN: str = os.environ.get("ANNOTATOR_LOGIN", "admin") ANNOTATOR_PASSWORD: str = os.environ.get("ANNOTATOR_PASSWORD", "admin") + # Connector credentials: Fernet key used to encrypt alert-API passwords at + # rest. Generate with: python -c "from cryptography.fernet import Fernet; + # print(Fernet.generate_key().decode())". Empty means connectors are + # disabled (create/update 400s, worker skips) — existing deployments that + # never set it keep working untouched. + CONNECTOR_SECRET_KEY: str = os.environ.get("CONNECTOR_SECRET_KEY", "") + # Where the worker reaches the annotation API to POST imported records. + # Not a secret. + ANNOTATION_API_INTERNAL_URL: str = os.environ.get( + "ANNOTATION_API_INTERNAL_URL", "http://annotation_api:5050" + ) + # Serving / DB pool sizing. # # Each uvicorn worker is a separate process with its own SQLAlchemy pool, diff --git a/annotation_api/src/app/main.py b/annotation_api/src/app/main.py index 19422d89..6d2f0eff 100644 --- a/annotation_api/src/app/main.py +++ b/annotation_api/src/app/main.py @@ -35,9 +35,10 @@ async def seed_default_users(session: AsyncSession) -> None: """Idempotent startup seeding: the human admin (AUTH_USERNAME) and the - login-disabled worker user (WORKER_USERNAME). The worker user no longer - writes annotations (the sweep is membership-only) but stays seeded — - existing machine-written annotations are attributed to it.""" + password-disabled worker user (WORKER_USERNAME). The worker no longer writes + annotations (the sweep is membership-only) but stays seeded for two reasons: + existing machine-written annotations are attributed to it, and it is the + identity the connector import mints its API token for (app/services/worker_auth.py).""" user_crud = UserCRUD(session) admin_user = await user_crud.get_by_username(settings.AUTH_USERNAME) @@ -69,10 +70,13 @@ async def seed_default_users(session: AsyncSession) -> None: worker_create = UserCreate( username=settings.WORKER_USERNAME, # Random and immediately discarded: this user exists purely - # for attribution and must never be able to log in (also - # seeded inactive, which login rejects independently). + # for attribution and can never log in, since no password can + # ever match. It must be active: app.api.dependencies.get_current_user + # is an alias for get_current_active_user, so an inactive + # identity could not call the endpoints the worker posts to + # (see app/services/worker_auth.py). password=secrets.token_urlsafe(32), - is_active=False, + is_active=True, is_superuser=False, can_localize=False, ) @@ -81,15 +85,12 @@ async def seed_default_users(session: AsyncSession) -> None: except Exception as e: logger.error(f"Failed to create worker user: {e}") await session.rollback() - elif worker_user.is_active: - # get-or-create adopted a pre-existing account: attribution will go - # to what looks like a human user. Almost certainly a naming - # collision — pick a different WORKER_USERNAME. - logger.warning( - f"User {settings.WORKER_USERNAME!r} already exists and is active; " - "the group-assignment sweep will attribute annotations to it." - ) else: + # With the worker seeded active, a get-or-create hit that adopted a + # pre-existing human account is no longer distinguishable from our + # own worker user — the is_active-based collision check this branch + # used to do is not expressible anymore. Not replacing it with a new + # heuristic; just noting the loss. logger.info("Worker user already exists") diff --git a/annotation_api/src/app/models.py b/annotation_api/src/app/models.py index 447b3f1b..2ca88abd 100644 --- a/annotation_api/src/app/models.py +++ b/annotation_api/src/app/models.py @@ -1,4 +1,4 @@ -from datetime import UTC, datetime +from datetime import UTC, date as date_type, datetime from enum import Enum from typing import List, Optional @@ -467,6 +467,119 @@ class User(SQLModel, table=True): ) +class ImportCoverageStatus(str, Enum): + """Outcome of one (connector, organization, day) import attempt.""" + + OK = "ok" # imported cleanly, including days with zero alerts + PARTIAL = "partial" # some alerts failed, some succeeded or were skipped + FAILED = "failed" # nothing imported: connector errored, or all alerts failed + + +class AlertApiConnector(SQLModel, table=True): + """A credentialed link to one alert API, imported daily by the worker.""" + + __tablename__ = "alert_api_connectors" + __table_args__ = ( + UniqueConstraint("base_url", name="uq_connector_base_url"), + # Sequence identity is (alert_api_id, source_api) and alert identity is + # (source_api, platform_alert_id). Two connectors sharing a source_api + # would let alert ids from different platforms collide. + UniqueConstraint("source_api", name="uq_connector_source_api"), + ) + + id: int = Field( + default=None, primary_key=True, sa_column_kwargs={"autoincrement": True} + ) + name: str = Field(max_length=100) + base_url: str = Field(max_length=255) + source_api: SourceApi + login: str = Field(max_length=100) + # Fernet token — see app.services.secrets. Never serialized to clients. + password_encrypted: str + is_enabled: bool = Field(default=True) + # Days re-imported on every run. This is also the catch-up mechanism: a + # missed run is recovered by the next run's window. + trailing_days: int = Field(default=3) + # "url" / "bucket-copy" / None = the importer's per-source auto-detect. + image_transfer: Optional[str] = Field(default=None, max_length=20) + last_verified_at: Optional[datetime] = Field( + default=None, sa_column=Column(DateTime(timezone=True)) + ) + last_verify_error: Optional[str] = Field(default=None) + created_at: datetime = Field( + default_factory=lambda: datetime.now(UTC), + sa_column=Column(DateTime(timezone=True)), + ) + updated_at: Optional[datetime] = Field( + default=None, + sa_column=Column(DateTime(timezone=True), onupdate=lambda: datetime.now(UTC)), + ) + + +class AlertApiConnectorOrganization(SQLModel, table=True): + """An organization discovered on a connector's alert API.""" + + __tablename__ = "alert_api_connector_organizations" + __table_args__ = ( + UniqueConstraint("connector_id", "organization_id", name="uq_connector_org"), + ) + + id: int = Field( + default=None, primary_key=True, sa_column_kwargs={"autoincrement": True} + ) + connector_id: int = Field( + sa_column=Column(ForeignKey("alert_api_connectors.id", ondelete="CASCADE")) + ) + # The organization's id on the REMOTE alert API, not a local FK. + organization_id: int + name: str = Field(max_length=200) + is_enabled: bool = Field(default=False) + enabled_at: Optional[datetime] = Field( + default=None, sa_column=Column(DateTime(timezone=True)) + ) + + +class AlertApiImportCoverage(SQLModel, table=True): + """One row per heatmap cell: what a connector imported for one organization + on one day. + + A day with zero alerts still gets a row (status ok, counts 0) — that is what + separates "we looked, nothing was there" from "we never got there". + """ + + __tablename__ = "alert_api_import_coverage" + __table_args__ = ( + UniqueConstraint( + "connector_id", + "organization_id", + "covered_date", + name="uq_coverage_connector_org_date", + ), + Index("ix_coverage_connector_date", "connector_id", "covered_date"), + ) + + id: int = Field( + default=None, primary_key=True, sa_column_kwargs={"autoincrement": True} + ) + connector_id: int = Field( + sa_column=Column(ForeignKey("alert_api_connectors.id", ondelete="CASCADE")) + ) + organization_id: int + covered_date: date_type + status: ImportCoverageStatus + alerts_fetched: int = Field(default=0) + alerts_imported: int = Field(default=0) + alerts_skipped: int = Field(default=0) + alerts_failed: int = Field(default=0) + # Object-split fan-out: one alert can become several annotation sequences. + lanes_created: int = Field(default=0) + error: Optional[str] = Field(default=None) + last_attempt_at: datetime = Field( + default_factory=lambda: datetime.now(UTC), + sa_column=Column(DateTime(timezone=True)), + ) + + class SequenceAnnotationContribution(SQLModel, table=True): __tablename__ = "sequence_annotation_contributions" __table_args__ = ( diff --git a/annotation_api/src/app/schemas/connector.py b/annotation_api/src/app/schemas/connector.py new file mode 100644 index 00000000..fc1dc084 --- /dev/null +++ b/annotation_api/src/app/schemas/connector.py @@ -0,0 +1,122 @@ +# Copyright (C) 2025, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +from datetime import date, datetime +from typing import Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from app.models import ImportCoverageStatus, SourceApi + +__all__ = [ + "ConnectorCreate", + "ConnectorOrganizationRead", + "ConnectorOrganizationUpdate", + "ConnectorRead", + "ConnectorTestRequest", + "ConnectorTestResult", + "ConnectorUpdate", + "CoverageCellRead", + "VerifyResult", +] + +ImageTransfer = Literal["url", "bucket-copy"] + + +class ConnectorCreate(BaseModel): + name: str = Field(max_length=100) + base_url: str = Field(max_length=255) + source_api: SourceApi + login: str = Field(max_length=100) + password: str = Field(min_length=1) + is_enabled: bool = True + trailing_days: int = Field(default=3, ge=1, le=30) + image_transfer: Optional[ImageTransfer] = None + + +class ConnectorUpdate(BaseModel): + name: Optional[str] = Field(default=None, max_length=100) + base_url: Optional[str] = Field(default=None, max_length=255) + login: Optional[str] = Field(default=None, max_length=100) + # Omitted means "leave the stored credential alone". + password: Optional[str] = Field(default=None, min_length=1) + is_enabled: Optional[bool] = None + trailing_days: Optional[int] = Field(default=None, ge=1, le=30) + image_transfer: Optional[ImageTransfer] = None + + +class ConnectorRead(BaseModel): + """Read model. Deliberately has no password field of any kind.""" + + id: int + name: str + base_url: str + source_api: SourceApi + login: str + has_password: bool + is_enabled: bool + trailing_days: int + image_transfer: Optional[str] + last_verified_at: Optional[datetime] + last_verify_error: Optional[str] + organizations_total: int = 0 + organizations_enabled: int = 0 + + model_config = ConfigDict(from_attributes=True) + + +class ConnectorOrganizationRead(BaseModel): + id: int + organization_id: int + name: str + is_enabled: bool + enabled_at: Optional[datetime] + + model_config = ConfigDict(from_attributes=True) + + +class ConnectorOrganizationUpdate(BaseModel): + is_enabled: bool + + +class CoverageCellRead(BaseModel): + organization_id: int + covered_date: date + status: ImportCoverageStatus + alerts_fetched: int + alerts_imported: int + alerts_skipped: int + alerts_failed: int + lanes_created: int + error: Optional[str] + + model_config = ConfigDict(from_attributes=True) + + +class ConnectorTestRequest(BaseModel): + """Stateless pre-save credential check. The plaintext password lives only + in this request body — never logged, never persisted.""" + + base_url: str = Field(max_length=255) + login: str = Field(max_length=100) + password: str = Field(min_length=1) + + +class ConnectorTestResult(BaseModel): + ok: bool + error: Optional[str] = None + organizations_total: int = 0 + + +class VerifyResult(BaseModel): + ok: bool + error: Optional[str] = None + organizations: list[ConnectorOrganizationRead] = [] + # Cross-organization probe: how many distinct organizations appeared in a + # sample listing, out of how many the connector can see. Reported as a count, + # not a boolean — one organization on a quiet day proves nothing. + organizations_seen_in_sample: int = 0 + organizations_total: int = 0 + sample_date: Optional[date] = None diff --git a/annotation_api/src/app/services/connector_import.py b/annotation_api/src/app/services/connector_import.py new file mode 100644 index 00000000..96073296 --- /dev/null +++ b/annotation_api/src/app/services/connector_import.py @@ -0,0 +1,229 @@ +# Copyright (C) 2025, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +"""Run one connector's daily import and record what it covered.""" + +import asyncio +import logging +from datetime import UTC, date, datetime, timedelta + +from sqlalchemy import select +from sqlmodel.ext.asyncio.session import AsyncSession + +from app.core.config import settings +from app.models import ( + AlertApiConnector, + AlertApiConnectorOrganization, + AlertApiImportCoverage, + ImportCoverageStatus, +) +from app.models import Sequence as SequenceModel +from app.models import SourceApi +from app.services.secrets import SecretKeyMissingError, decrypt_secret +from app.services.worker_auth import mint_worker_token +from scripts.data_transfer.ingestion.alert_api.runner import ( + ImportConfig, + OrganizationStats, + run_import, +) + +logger = logging.getLogger(__name__) + +__all__ = ["build_skip_ids", "import_connector"] + + +async def build_skip_ids(session: AsyncSession, source_api: SourceApi) -> set[int]: + """Alert-API sequence ids we already hold for this platform. + + platform_alert_id is the alert API's own sequence id, shared by every lane of + an alert (object-split siblings included), and indexed as + ix_sequence_platform_alert_id — so this is one cheap query, and the result + lets the importer skip alerts before fetching any of their detections. + """ + result = await session.execute( + select(SequenceModel.platform_alert_id) + .where(SequenceModel.source_api == source_api) + .distinct() + ) + return {row for row in result.scalars().all() if row is not None} + + +def _status(stats: OrganizationStats) -> ImportCoverageStatus: + if stats.alerts_failed and not (stats.alerts_imported or stats.alerts_skipped): + return ImportCoverageStatus.FAILED + if stats.alerts_failed: + return ImportCoverageStatus.PARTIAL + return ImportCoverageStatus.OK + + +async def _upsert_coverage( + session: AsyncSession, + *, + connector_id: int, + organization_id: int, + covered_date: date, + status: ImportCoverageStatus, + stats: OrganizationStats, + error: str | None, +) -> None: + row = ( + await session.execute( + select(AlertApiImportCoverage).where( + AlertApiImportCoverage.connector_id == connector_id, + AlertApiImportCoverage.organization_id == organization_id, + AlertApiImportCoverage.covered_date == covered_date, + ) + ) + ).scalar_one_or_none() + if row is None: + row = AlertApiImportCoverage( + connector_id=connector_id, + organization_id=organization_id, + covered_date=covered_date, + ) + row.status = status + row.alerts_fetched = stats.alerts_fetched + row.alerts_imported = stats.alerts_imported + row.alerts_skipped = stats.alerts_skipped + row.alerts_failed = stats.alerts_failed + row.lanes_created = stats.lanes_created + row.error = error + row.last_attempt_at = datetime.now(UTC) + session.add(row) + + +async def import_connector( + session: AsyncSession, + connector: AlertApiConnector, + *, + today: date, +) -> None: + """Import the connector's trailing window and write one coverage row per + enabled organization per day. + + Never raises: a connector that cannot run must not take down the sweep for + the others. Failures are recorded as coverage rows where that is + meaningful; a DB-layer failure that leaves us without enough information to + attribute a row correctly is logged and swallowed instead, mirroring the + missing-CONNECTOR_SECRET_KEY case below. The whole function is guarded, not + just the outbound `run_import` call, so a database hiccup anywhere in here + (listing organizations, minting the worker token, reading/writing the skip + set or coverage rows) can never propagate out to the caller. + """ + try: + organizations = ( + ( + await session.execute( + select(AlertApiConnectorOrganization).where( + AlertApiConnectorOrganization.connector_id == connector.id, + AlertApiConnectorOrganization.is_enabled.is_(True), + ) + ) + ) + .scalars() + .all() + ) + except Exception: # noqa: BLE001 - logged, not raised + logger.exception( + "connector %s: failed to load organizations; skipping", connector.id + ) + await session.rollback() + return + + if not organizations: + logger.info("connector %s has no enabled organizations; skipping", connector.id) + return + + try: + password = decrypt_secret(connector.password_encrypted) + except SecretKeyMissingError as exc: + # No coverage rows: this is a deployment problem, not a data gap, and + # writing "failed" cells would misattribute it to the alert API. + logger.error("connector %s cannot be decrypted: %s", connector.id, exc) + return + + try: + token = await mint_worker_token(session) + except Exception: # noqa: BLE001 - logged, not raised + logger.exception( + "connector %s: failed to mint a worker token; skipping", connector.id + ) + await session.rollback() + return + if token is None: + logger.error("connector %s: no worker token available; skipping", connector.id) + return + + org_ids = {org.organization_id for org in organizations} + try: + skip_ids = await build_skip_ids(session, connector.source_api) + except Exception: # noqa: BLE001 - logged, not raised + logger.exception( + "connector %s: failed to load the skip set; skipping", connector.id + ) + await session.rollback() + return + + # today is excluded: the day is still in progress on the alert API. + days = [ + today - timedelta(days=offset) + for offset in range(connector.trailing_days, 0, -1) + ] + + for day in days: + config = ImportConfig( + alert_api_url=connector.base_url, + login=connector.login, + password=password, + admin_login=connector.login, + admin_password=password, + annotation_api_url=settings.ANNOTATION_API_INTERNAL_URL, + annotation_api_token=token, + date_from=day, + date_end=day, + source_api=connector.source_api.value, + image_transfer=connector.image_transfer, + organization_ids=org_ids, + skip_platform_alert_ids=frozenset(skip_ids), + ) + error: str | None = None + per_org: dict[int, OrganizationStats] = {} + try: + result = await asyncio.to_thread(run_import, config) + per_org = result.per_organization + error = result.error + except Exception as exc: # noqa: BLE001 - recorded, not raised + logger.exception("connector %s import failed for %s", connector.id, day) + error = f"{type(exc).__name__}: {exc}" + + # Coverage bookkeeping for this day: writing the rows, committing, and + # re-deriving the skip set for the next day are one unit — if any part + # of it fails, roll back and stop the sweep for this connector rather + # than risk the next day's write landing on top of an aborted + # transaction. + try: + for org in organizations: + stats = per_org.get(org.organization_id, OrganizationStats()) + status = ImportCoverageStatus.FAILED if error else _status(stats) + await _upsert_coverage( + session, + connector_id=connector.id, + organization_id=org.organization_id, + covered_date=day, + status=status, + stats=stats, + error=error, + ) + await session.commit() + + # Alerts imported for this day must not be re-fetched on the next + # day in the window. + skip_ids = await build_skip_ids(session, connector.source_api) + except Exception: # noqa: BLE001 - logged, not raised + logger.exception( + "connector %s: coverage bookkeeping failed for %s", connector.id, day + ) + await session.rollback() + return diff --git a/annotation_api/src/app/services/connector_verify.py b/annotation_api/src/app/services/connector_verify.py new file mode 100644 index 00000000..25ede4c3 --- /dev/null +++ b/annotation_api/src/app/services/connector_verify.py @@ -0,0 +1,220 @@ +# Copyright (C) 2025, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +"""Verify a connector: prove the credential works, discover the organizations it +can see, and measure how many of them actually appear in a sample listing. + +The last part matters because the whole connector design assumes one admin +account can list sequences across every organization. That assumption is reported +as a count rather than asserted as a boolean: seeing one organization on a quiet +day proves nothing, but seeing four of seven proves cross-org listing works. +""" + +import asyncio +import logging +from datetime import UTC, date, datetime, timedelta +from typing import Any + +from sqlalchemy import select +from sqlmodel.ext.asyncio.session import AsyncSession + +from app.models import AlertApiConnector, AlertApiConnectorOrganization +from app.schemas.connector import ( + ConnectorOrganizationRead, + ConnectorTestResult, + VerifyResult, +) +from app.services.secrets import decrypt_secret +from scripts.data_transfer.ingestion.alert_api import client as alert_api_client + +logger = logging.getLogger(__name__) + +_PROBE_LIMIT = 200 + +# _probe makes 4 sequential HTTP calls: the token exchange (client.py's own +# 5s timeout) plus three list endpoints, each bounded at 30s by api_get's +# timeout. 95s covers that worst-case sum with a small buffer, so a probe +# where every call is legitimately slow-but-working still completes; a probe +# against a host that black-holes packets was previously unbounded (the +# asyncio default-executor thread it occupies would never return) and is now +# bounded here too. +_PROBE_TIMEOUT_SECONDS = 100 + +# The pre-save credential check makes 2 sequential HTTP calls (token exchange, +# 5s bound; one list endpoint, 30s bound). 25s keeps the backend's answer +# ahead of the frontend's global 30s axios timeout, so the browser never +# gives up before the server has spoken. +_TEST_TIMEOUT_SECONDS = 25 + +__all__ = ["check_connector_credentials", "verify_connector"] + + +def _require_list(call: str, response: Any) -> None: + """`api_get` only raises when the body fails to parse as JSON — a non-2xx + response with a valid JSON error body (e.g. {"detail": "..."}) comes back + as a dict where a list is expected. Fail with the alert API's own detail: + for a non-admin credential that detail is "Incompatible token scope.", + the one hint telling the operator to swap in an admin account.""" + if not isinstance(response, list): + detail = response.get("detail") if isinstance(response, dict) else None + raise ValueError( + f"alert API returned an unexpected {call} response" + + (f": {detail}" if detail else "") + ) + + +def _probe( + base_url: str, login: str, password: str, sample_date: date +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]]]: + """Blocking: token, organizations, cameras, and one day of sequences. + + The alert API client is synchronous `requests`; the caller wraps this in a + thread so the event loop stays free. + """ + token = alert_api_client.get_api_access_token( + api_endpoint=base_url, username=login, password=password + ) + organizations = alert_api_client.list_organizations( + api_endpoint=base_url, access_token=token + ) + cameras = alert_api_client.list_cameras(api_endpoint=base_url, access_token=token) + sequences = alert_api_client.list_sequences_for_date( + api_endpoint=base_url, + date=sample_date, + limit=_PROBE_LIMIT, + offset=0, + access_token=token, + risk_score="extreme", + ) + return organizations, cameras, sequences + + +def _probe_credentials(base_url: str, login: str, password: str) -> int: + """Blocking: token exchange, then the organizations listing — the two + calls that cover both real failure modes (wrong password, org-scoped + credential). No sequence probe: that stays verify's job.""" + token = alert_api_client.get_api_access_token( + api_endpoint=base_url, username=login, password=password + ) + organizations = alert_api_client.list_organizations( + api_endpoint=base_url, access_token=token + ) + _require_list("organizations", organizations) + return len(organizations) + + +async def check_connector_credentials( + base_url: str, login: str, password: str +) -> ConnectorTestResult: + """Stateless pre-save credential check: no DB, nothing persisted, never + raises for an unreachable or unauthorized alert API — this runs behind a + button a human is watching.""" + try: + organizations_total = await asyncio.wait_for( + asyncio.to_thread(_probe_credentials, base_url, login, password), + timeout=_TEST_TIMEOUT_SECONDS, + ) + except Exception as exc: # noqa: BLE001 - surfaced to the operator verbatim + return ConnectorTestResult(ok=False, error=f"{type(exc).__name__}: {exc}") + return ConnectorTestResult(ok=True, organizations_total=organizations_total) + + +async def verify_connector( + session: AsyncSession, + connector: AlertApiConnector, + *, + today: date, +) -> VerifyResult: + """Authenticate, upsert discovered organizations, and probe cross-org reach. + + Never raises for an unreachable or unauthorized alert API — the failure is + recorded on the connector and returned, because this runs behind a button a + human is watching. + """ + sample_date = today - timedelta(days=1) + try: + password = decrypt_secret(connector.password_encrypted) + organizations, cameras, sequences = await asyncio.wait_for( + asyncio.to_thread( + _probe, connector.base_url, connector.login, password, sample_date + ), + timeout=_PROBE_TIMEOUT_SECONDS, + ) + for call, response in ( + ("organizations", organizations), + ("cameras", cameras), + ("sequences", sequences), + ): + _require_list(call, response) + + existing = { + row.organization_id: row + for row in ( + ( + await session.execute( + select(AlertApiConnectorOrganization).where( + AlertApiConnectorOrganization.connector_id == connector.id + ) + ) + ) + .scalars() + .all() + ) + } + for org in organizations: + row = existing.get(org["id"]) + if row is None: + # New organizations start disabled: discovery must never + # silently widen what gets ingested. + row = AlertApiConnectorOrganization( + connector_id=connector.id, + organization_id=org["id"], + name=org.get("name") or str(org["id"]), + is_enabled=False, + ) + else: + row.name = org.get("name") or row.name + session.add(row) + + camera_org = {c["id"]: c.get("organization_id") for c in cameras} + seen = { + camera_org.get(seq.get("camera_id")) + for seq in sequences + if camera_org.get(seq.get("camera_id")) is not None + } + except Exception as exc: # noqa: BLE001 - surfaced to the operator verbatim + # str(exc) is safe: the client raises on status codes and never embeds + # the request body, so the password cannot leak into this message. + await session.rollback() + message = f"{type(exc).__name__}: {exc}" + connector.last_verify_error = message + session.add(connector) + await session.commit() + logger.warning("connector %s verification failed: %s", connector.id, message) + return VerifyResult(ok=False, error=message) + + connector.last_verified_at = datetime.now(UTC) + connector.last_verify_error = None + session.add(connector) + await session.commit() + + rows = ( + ( + await session.execute( + select(AlertApiConnectorOrganization) + .where(AlertApiConnectorOrganization.connector_id == connector.id) + .order_by(AlertApiConnectorOrganization.name) + ) + ) + .scalars() + .all() + ) + return VerifyResult( + ok=True, + organizations=[ConnectorOrganizationRead.model_validate(r) for r in rows], + organizations_seen_in_sample=len(seen), + organizations_total=len(organizations), + sample_date=sample_date, + ) diff --git a/annotation_api/src/app/services/secrets.py b/annotation_api/src/app/services/secrets.py new file mode 100644 index 00000000..9beef9ba --- /dev/null +++ b/annotation_api/src/app/services/secrets.py @@ -0,0 +1,55 @@ +# Copyright (C) 2025, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +"""Symmetric encryption for credentials the system must be able to replay. + +Alert-API passwords cannot be hashed: we need the plaintext to log in. They are +therefore encrypted with a key held outside the database, so a database dump — a +backup, a copy pulled for debugging — carries nothing usable on its own. +""" + +from cryptography.fernet import Fernet, InvalidToken + +from app.core.config import settings + +__all__ = ["SecretKeyMissingError", "decrypt_secret", "encrypt_secret"] + +_KEY_HELP = ( + "CONNECTOR_SECRET_KEY is not set to a valid Fernet key. Generate one with: " + 'python -c "from cryptography.fernet import Fernet; ' + 'print(Fernet.generate_key().decode())"' +) + + +class SecretKeyMissingError(RuntimeError): + """CONNECTOR_SECRET_KEY is unset, malformed, or does not match a token.""" + + +def _fernet() -> Fernet: + # Read settings at call time, not import time, so tests and runtime + # reconfiguration both work. + key = settings.CONNECTOR_SECRET_KEY + if not key: + raise SecretKeyMissingError(_KEY_HELP) + try: + return Fernet(key.encode()) + except (ValueError, TypeError) as exc: + raise SecretKeyMissingError(_KEY_HELP) from exc + + +def encrypt_secret(plaintext: str) -> str: + return _fernet().encrypt(plaintext.encode()).decode() + + +def decrypt_secret(token: str) -> str: + try: + return _fernet().decrypt(token.encode()).decode() + except InvalidToken as exc: + # Never echo the token or any plaintext. + raise SecretKeyMissingError( + "Stored credential could not be decrypted with the current " + "CONNECTOR_SECRET_KEY. If the key was rotated or lost, re-enter the " + "connector credentials." + ) from exc diff --git a/annotation_api/src/app/services/worker_auth.py b/annotation_api/src/app/services/worker_auth.py new file mode 100644 index 00000000..ff28df94 --- /dev/null +++ b/annotation_api/src/app/services/worker_auth.py @@ -0,0 +1,39 @@ +# Copyright (C) 2025, Pyronear. + +# This program is licensed under the Apache License 2.0. +# See LICENSE or go to for full license details. + +"""The worker's identity when it calls the annotation API. + +create_access_token is a pure function over JWT_SECRET, which the worker already +has, and the worker already resolves the worker user by name. So it mints its own +token rather than carrying a password: no plaintext credential in the worker's +environment, no credential duplicated to talk to itself, and no cold-boot race +where the worker starts before the API seeds its users. +""" + +import logging + +from sqlmodel.ext.asyncio.session import AsyncSession + +from app.auth.dependencies import create_access_token +from app.core.config import settings +from app.crud import UserCRUD + +logger = logging.getLogger(__name__) + +__all__ = ["mint_worker_token"] + + +async def mint_worker_token(session: AsyncSession) -> str | None: + """A bearer token for the seeded worker user, or None if it does not exist + yet (the API seeds it at startup; a very early worker run can lose that race, + and the next scheduled run will succeed).""" + user = await UserCRUD(session).get_by_username(settings.WORKER_USERNAME) + if user is None: + logger.warning( + "mint_worker_token: worker user %r not found; skipping", + settings.WORKER_USERNAME, + ) + return None + return create_access_token(data={"sub": user.username, "user_id": user.id}) diff --git a/annotation_api/src/app/worker.py b/annotation_api/src/app/worker.py index a2293534..bf834335 100644 --- a/annotation_api/src/app/worker.py +++ b/annotation_api/src/app/worker.py @@ -23,9 +23,10 @@ from app.core.config import settings from app.db import engine -from app.models import Detection +from app.models import AlertApiConnector, Detection from app.models import Sequence as SequenceModel from app.services.auto_annotate_scheduling import schedule_pending_auto_annotate +from app.services.connector_import import import_connector from app.services.group_assignment import assign_ungrouped_sequences from app.services.smoke_detector import ( SmokeDetector, @@ -180,3 +181,46 @@ async def schedule_auto_annotate(timestamp: int) -> None: len(sequence_ids), sequence_ids, ) + + +@app.periodic(cron="0 3 * * *") +@app.task(name="schedule_connector_imports", queueing_lock="schedule_connector_imports") +async def schedule_connector_imports(timestamp: int) -> None: + """Daily sweep: defer one import job per enabled connector. + + No "already ran today" bookkeeping is needed — procrastinate defers a + periodic task once per cron period, and each job's own queueing_lock stops a + still-running connector from stacking up a second job. + + Not hourly with a per-connector run hour: that would only buy staggering. + What makes the schedule robust is trailing_days — a worker down at 03:00 + loses nothing, because the next run re-covers that date inside its window. + """ + async with AsyncSession(engine, expire_on_commit=False) as session: + connectors = ( + ( + await session.execute( + select(AlertApiConnector).where( + AlertApiConnector.is_enabled.is_(True) + ) + ) + ) + .scalars() + .all() + ) + for connector in connectors: + await run_connector_import.configure( + queueing_lock=f"connector-import-{connector.id}" + ).defer_async(connector_id=connector.id) + logger.info("schedule_connector_imports: deferred %d connector(s)", len(connectors)) + + +@app.task(name="run_connector_import") +async def run_connector_import(connector_id: int) -> None: + """Import one connector's trailing window.""" + async with AsyncSession(engine, expire_on_commit=False) as session: + connector = await session.get(AlertApiConnector, connector_id) + if connector is None: + logger.warning("run_connector_import: connector %s gone", connector_id) + return + await import_connector(session, connector, today=datetime.now(UTC).date()) diff --git a/annotation_api/src/migrations/versions/2026_08_11_1000-d6e7f8a9b0c1_add_alert_api_connectors.py b/annotation_api/src/migrations/versions/2026_08_11_1000-d6e7f8a9b0c1_add_alert_api_connectors.py new file mode 100644 index 00000000..18fb816b --- /dev/null +++ b/annotation_api/src/migrations/versions/2026_08_11_1000-d6e7f8a9b0c1_add_alert_api_connectors.py @@ -0,0 +1,118 @@ +"""Add alert API connector, organization, and import coverage tables + +Revision ID: d6e7f8a9b0c1 +Revises: c5d6e7f8a9b0 +Create Date: 2026-08-11 10:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "d6e7f8a9b0c1" +down_revision = "c5d6e7f8a9b0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "alert_api_connectors", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("name", sa.String(length=100), nullable=False), + sa.Column("base_url", sa.String(length=255), nullable=False), + sa.Column( + "source_api", + postgresql.ENUM( + "PYRONEAR_FRENCH_API", + "ALERT_WILDFIRE", + "CENIA", + name="sourceapi", + create_type=False, + ), + nullable=False, + ), + sa.Column("login", sa.String(length=100), nullable=False), + sa.Column("password_encrypted", sa.String(), nullable=False), + sa.Column("is_enabled", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("trailing_days", sa.Integer(), nullable=False, server_default="3"), + sa.Column("image_transfer", sa.String(length=20), nullable=True), + sa.Column("last_verified_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_verify_error", sa.String(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("base_url", name="uq_connector_base_url"), + sa.UniqueConstraint("source_api", name="uq_connector_source_api"), + ) + + op.create_table( + "alert_api_connector_organizations", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("connector_id", sa.Integer(), nullable=False), + sa.Column("organization_id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=200), nullable=False), + sa.Column( + "is_enabled", sa.Boolean(), nullable=False, server_default=sa.false() + ), + sa.Column("enabled_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["connector_id"], ["alert_api_connectors.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("connector_id", "organization_id", name="uq_connector_org"), + ) + + op.create_table( + "alert_api_import_coverage", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("connector_id", sa.Integer(), nullable=False), + sa.Column("organization_id", sa.Integer(), nullable=False), + sa.Column("covered_date", sa.Date(), nullable=False), + sa.Column( + "status", + postgresql.ENUM("OK", "PARTIAL", "FAILED", name="importcoveragestatus"), + nullable=False, + ), + sa.Column("alerts_fetched", sa.Integer(), nullable=False, server_default="0"), + sa.Column("alerts_imported", sa.Integer(), nullable=False, server_default="0"), + sa.Column("alerts_skipped", sa.Integer(), nullable=False, server_default="0"), + sa.Column("alerts_failed", sa.Integer(), nullable=False, server_default="0"), + sa.Column("lanes_created", sa.Integer(), nullable=False, server_default="0"), + sa.Column("error", sa.String(), nullable=True), + sa.Column( + "last_attempt_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["connector_id"], ["alert_api_connectors.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "connector_id", + "organization_id", + "covered_date", + name="uq_coverage_connector_org_date", + ), + ) + op.create_index( + "ix_coverage_connector_date", + "alert_api_import_coverage", + ["connector_id", "covered_date"], + ) + + +def downgrade() -> None: + op.drop_index("ix_coverage_connector_date", table_name="alert_api_import_coverage") + op.drop_table("alert_api_import_coverage") + op.drop_table("alert_api_connector_organizations") + op.drop_table("alert_api_connectors") + sa.Enum(name="importcoveragestatus").drop(op.get_bind(), checkfirst=True) diff --git a/annotation_api/src/tests/conftest.py b/annotation_api/src/tests/conftest.py index 4f4615d3..49e15eca 100644 --- a/annotation_api/src/tests/conftest.py +++ b/annotation_api/src/tests/conftest.py @@ -268,12 +268,13 @@ async def inactive_user(async_session: AsyncSession) -> User: @pytest_asyncio.fixture(scope="function") async def worker_user(async_session: AsyncSession) -> User: - """Create the seeded system worker user (login-disabled attribution account).""" + """Create the seeded system worker user (password-disabled attribution + account; active so it can call the API — see app/main.py).""" user_crud = UserCRUD(async_session) user_create = UserCreate( username=settings.WORKER_USERNAME, password="workerpassword123", - is_active=False, + is_active=True, is_superuser=False, ) user = await user_crud.create_user(user_create) diff --git a/annotation_api/src/tests/crud/test_connector_models.py b/annotation_api/src/tests/crud/test_connector_models.py new file mode 100644 index 00000000..24af7123 --- /dev/null +++ b/annotation_api/src/tests/crud/test_connector_models.py @@ -0,0 +1,124 @@ +"""The three connector tables exist with the constraints the design relies on.""" + +from datetime import date + +import pytest +from sqlalchemy.exc import IntegrityError +from sqlmodel import select + +from app.models import ( + AlertApiConnector, + AlertApiConnectorOrganization, + AlertApiImportCoverage, + ImportCoverageStatus, + SourceApi, +) + + +async def _connector( + session, + *, + source_api=SourceApi.PYRONEAR_FRENCH_API, + base_url="https://a.example", +): + connector = AlertApiConnector( + name="Test", + base_url=base_url, + source_api=source_api, + login="admin", + password_encrypted="token", + ) + session.add(connector) + await session.commit() + await session.refresh(connector) + return connector + + +async def test_connector_defaults(async_session): + connector = await _connector(async_session) + assert connector.is_enabled is True + assert connector.trailing_days == 3 + assert connector.image_transfer is None + assert connector.last_verified_at is None + + +async def test_source_api_is_unique_across_connectors(async_session): + await _connector(async_session, base_url="https://a.example") + with pytest.raises(IntegrityError): + await _connector(async_session, base_url="https://b.example") + await async_session.rollback() + + +async def test_base_url_is_unique(async_session): + await _connector(async_session, base_url="https://a.example") + with pytest.raises(IntegrityError): + await _connector( + async_session, base_url="https://a.example", source_api=SourceApi.CENIA + ) + await async_session.rollback() + + +async def test_organization_unique_per_connector(async_session): + connector = await _connector(async_session) + async_session.add( + AlertApiConnectorOrganization( + connector_id=connector.id, organization_id=7, name="Ardeche" + ) + ) + await async_session.commit() + async_session.add( + AlertApiConnectorOrganization( + connector_id=connector.id, organization_id=7, name="Ardeche" + ) + ) + with pytest.raises(IntegrityError): + await async_session.commit() + await async_session.rollback() + + +async def test_coverage_unique_per_connector_org_date(async_session): + connector = await _connector(async_session) + for _ in range(2): + async_session.add( + AlertApiImportCoverage( + connector_id=connector.id, + organization_id=7, + covered_date=date(2026, 8, 5), + status=ImportCoverageStatus.OK, + ) + ) + with pytest.raises(IntegrityError): + await async_session.commit() + await async_session.rollback() + + +async def test_deleting_connector_cascades(async_session): + connector = await _connector(async_session) + async_session.add( + AlertApiConnectorOrganization( + connector_id=connector.id, organization_id=7, name="Ardeche" + ) + ) + async_session.add( + AlertApiImportCoverage( + connector_id=connector.id, + organization_id=7, + covered_date=date(2026, 8, 5), + status=ImportCoverageStatus.OK, + ) + ) + await async_session.commit() + + await async_session.delete(connector) + await async_session.commit() + + orgs = ( + (await async_session.execute(select(AlertApiConnectorOrganization))) + .scalars() + .all() + ) + coverage = ( + (await async_session.execute(select(AlertApiImportCoverage))).scalars().all() + ) + assert orgs == [] + assert coverage == [] diff --git a/annotation_api/src/tests/endpoints/test_connector_coverage.py b/annotation_api/src/tests/endpoints/test_connector_coverage.py new file mode 100644 index 00000000..e55786c3 --- /dev/null +++ b/annotation_api/src/tests/endpoints/test_connector_coverage.py @@ -0,0 +1,170 @@ +"""Organization toggling stamps enabled_at once; coverage reads are windowed.""" + +from datetime import UTC, date, datetime, timedelta + +import pytest +from cryptography.fernet import Fernet +from httpx import ASGITransport, AsyncClient + +from app.core.config import settings +from app.db import get_session +from app.main import app +from app.models import ( + AlertApiConnector, + AlertApiConnectorOrganization, + AlertApiImportCoverage, + ImportCoverageStatus, + SourceApi, +) +from app.services.secrets import encrypt_secret + + +@pytest.fixture +def secret_key(monkeypatch): + monkeypatch.setattr( + settings, "CONNECTOR_SECRET_KEY", Fernet.generate_key().decode() + ) + + +@pytest.fixture +async def connector(async_session, secret_key): + row = AlertApiConnector( + name="Test", + base_url="https://a.example", + source_api=SourceApi.PYRONEAR_FRENCH_API, + login="admin", + password_encrypted=encrypt_secret("pw"), + ) + async_session.add(row) + await async_session.commit() + await async_session.refresh(row) + async_session.add( + AlertApiConnectorOrganization( + connector_id=row.id, organization_id=1, name="Ardeche" + ) + ) + await async_session.commit() + return row + + +async def test_enabling_an_org_stamps_enabled_at(authenticated_client, connector): + response = await authenticated_client.patch( + f"/connectors/{connector.id}/organizations/1", json={"is_enabled": True} + ) + assert response.status_code == 200 + body = response.json() + assert body["is_enabled"] is True + assert body["enabled_at"] is not None + + +async def test_disabling_keeps_the_original_enabled_at(authenticated_client, connector): + first = ( + await authenticated_client.patch( + f"/connectors/{connector.id}/organizations/1", json={"is_enabled": True} + ) + ).json() + await authenticated_client.patch( + f"/connectors/{connector.id}/organizations/1", json={"is_enabled": False} + ) + again = ( + await authenticated_client.patch( + f"/connectors/{connector.id}/organizations/1", json={"is_enabled": True} + ) + ).json() + # enabled_at marks when the org FIRST entered ingestion — the heatmap uses it + # to grey out days that predate it, so re-enabling must not move it. + assert again["enabled_at"] == first["enabled_at"] + + +async def test_unknown_org_returns_404(authenticated_client, connector): + response = await authenticated_client.patch( + f"/connectors/{connector.id}/organizations/999", json={"is_enabled": True} + ) + assert response.status_code == 404 + assert ( + response.json()["detail"] + == "Organization not found on this connector; run verify first." + ) + + +async def test_coverage_is_filtered_by_window( + authenticated_client, connector, async_session +): + # 08-04 and 08-06 sit exactly on the window's bounds — they must be + # included, proving the filter is inclusive (>=/<=), not exclusive (>/<). + for day in ( + date(2026, 8, 1), + date(2026, 8, 4), + date(2026, 8, 5), + date(2026, 8, 6), + date(2026, 8, 9), + ): + async_session.add( + AlertApiImportCoverage( + connector_id=connector.id, + organization_id=1, + covered_date=day, + status=ImportCoverageStatus.OK, + alerts_imported=2, + ) + ) + await async_session.commit() + + response = await authenticated_client.get( + f"/connectors/{connector.id}/coverage", + params={"date_from": "2026-08-04", "date_end": "2026-08-06"}, + ) + assert response.status_code == 200 + body = response.json() + assert [cell["covered_date"] for cell in body] == [ + "2026-08-04", + "2026-08-05", + "2026-08-06", + ] + assert body[0]["alerts_imported"] == 2 + + +async def test_coverage_defaults_to_the_last_30_days( + authenticated_client, connector, async_session +): + # Derive expected bounds the same way the endpoint does (datetime.now(UTC).date()) + # rather than hardcoding dates, so the test is timezone-independent. + today = datetime.now(UTC).date() + window_start = today - timedelta(days=29) + just_outside_window = today - timedelta(days=30) + for day in (just_outside_window, window_start, today): + async_session.add( + AlertApiImportCoverage( + connector_id=connector.id, + organization_id=1, + covered_date=day, + status=ImportCoverageStatus.OK, + alerts_imported=1, + ) + ) + await async_session.commit() + + response = await authenticated_client.get(f"/connectors/{connector.id}/coverage") + assert response.status_code == 200 + covered_dates = [cell["covered_date"] for cell in response.json()] + assert covered_dates == [window_start.isoformat(), today.isoformat()] + assert just_outside_window.isoformat() not in covered_dates + + +async def test_regular_user_cannot_read_coverage( + async_session, regular_user_token, connector +): + async def get_test_session(): + yield async_session + + app.dependency_overrides[get_session] = get_test_session + async with AsyncClient( + transport=ASGITransport(app=app), + base_url=f"http://api.localhost:8050{settings.API_V1_STR}", + headers={"Authorization": f"Bearer {regular_user_token}"}, + follow_redirects=True, + timeout=5, + ) as client: + response = await client.get(f"/connectors/{connector.id}/coverage") + app.dependency_overrides.clear() + assert response.status_code == 403 diff --git a/annotation_api/src/tests/endpoints/test_connectors.py b/annotation_api/src/tests/endpoints/test_connectors.py new file mode 100644 index 00000000..2f56103f --- /dev/null +++ b/annotation_api/src/tests/endpoints/test_connectors.py @@ -0,0 +1,194 @@ +"""Connector CRUD: superuser-only, password write-only, and a clear 400 when +CONNECTOR_SECRET_KEY is unset.""" + +import pytest +from cryptography.fernet import Fernet +from httpx import ASGITransport, AsyncClient +from sqlmodel import select + +from app.core.config import settings +from app.db import get_session +from app.main import app +from app.models import AlertApiConnector +from app.services import connector_verify +from app.services.secrets import decrypt_secret + +PAYLOAD = { + "name": "Pyronear France", + "base_url": "https://alertapi.pyronear.org", + "source_api": "pyronear_french", + "login": "admin", + "password": "hunter2", +} + + +@pytest.fixture +def secret_key(monkeypatch): + monkeypatch.setattr( + settings, "CONNECTOR_SECRET_KEY", Fernet.generate_key().decode() + ) + + +@pytest.fixture +async def regular_client(async_session, regular_user_token): + async def get_test_session(): + yield async_session + + app.dependency_overrides[get_session] = get_test_session + async with AsyncClient( + transport=ASGITransport(app=app), + base_url=f"http://api.localhost:8050{settings.API_V1_STR}", + headers={"Authorization": f"Bearer {regular_user_token}"}, + follow_redirects=True, + timeout=5, + ) as client: + yield client + app.dependency_overrides.clear() + + +async def test_create_returns_connector_without_password( + authenticated_client, secret_key +): + response = await authenticated_client.post("/connectors/", json=PAYLOAD) + assert response.status_code == 201 + body = response.json() + assert body["name"] == "Pyronear France" + assert body["has_password"] is True + assert "password" not in body + assert "password_encrypted" not in body + assert "hunter2" not in response.text + + +async def test_password_is_encrypted_at_rest( + authenticated_client, secret_key, async_session +): + await authenticated_client.post("/connectors/", json=PAYLOAD) + connector = (await async_session.execute(select(AlertApiConnector))).scalars().one() + assert connector.password_encrypted != "hunter2" + assert decrypt_secret(connector.password_encrypted) == "hunter2" + + +async def test_create_without_secret_key_returns_400(authenticated_client, monkeypatch): + monkeypatch.setattr(settings, "CONNECTOR_SECRET_KEY", "") + response = await authenticated_client.post("/connectors/", json=PAYLOAD) + assert response.status_code == 400 + assert "CONNECTOR_SECRET_KEY" in response.json()["detail"] + + +async def test_regular_user_cannot_list_connectors(regular_client): + assert (await regular_client.get("/connectors/")).status_code == 403 + + +async def test_regular_user_cannot_create_connector(regular_client): + assert (await regular_client.post("/connectors/", json=PAYLOAD)).status_code == 403 + + +async def test_list_never_leaks_password(authenticated_client, secret_key): + await authenticated_client.post("/connectors/", json=PAYLOAD) + response = await authenticated_client.get("/connectors/") + assert response.status_code == 200 + assert "hunter2" not in response.text + assert response.json()[0]["has_password"] is True + + +async def test_patch_without_password_keeps_existing( + authenticated_client, secret_key, async_session +): + created = (await authenticated_client.post("/connectors/", json=PAYLOAD)).json() + response = await authenticated_client.patch( + f"/connectors/{created['id']}", json={"trailing_days": 7} + ) + assert response.status_code == 200 + assert response.json()["trailing_days"] == 7 + + connector = (await async_session.execute(select(AlertApiConnector))).scalars().one() + assert decrypt_secret(connector.password_encrypted) == "hunter2" + + +async def test_patch_with_password_replaces_it( + authenticated_client, secret_key, async_session +): + created = (await authenticated_client.post("/connectors/", json=PAYLOAD)).json() + await authenticated_client.patch( + f"/connectors/{created['id']}", json={"password": "newpass"} + ) + connector = (await async_session.execute(select(AlertApiConnector))).scalars().one() + assert decrypt_secret(connector.password_encrypted) == "newpass" + + +async def test_duplicate_source_api_is_rejected(authenticated_client, secret_key): + await authenticated_client.post("/connectors/", json=PAYLOAD) + duplicate = {**PAYLOAD, "base_url": "https://other.example"} + response = await authenticated_client.post("/connectors/", json=duplicate) + assert response.status_code == 409 + + +async def test_delete_removes_connector(authenticated_client, secret_key): + created = (await authenticated_client.post("/connectors/", json=PAYLOAD)).json() + deleted = await authenticated_client.delete(f"/connectors/{created['id']}") + assert deleted.status_code == 204 + assert (await authenticated_client.get("/connectors/")).json() == [] + + +# --- POST /connectors/test: stateless pre-save credential check --- + +TEST_PAYLOAD = { + "base_url": "https://a.example", + "login": "admin", + "password": "good", +} + + +@pytest.fixture +def stub_alert_api(monkeypatch): + def fake_token(api_endpoint, username, password): + if password != "good": + raise RuntimeError("401 Unauthorized") + return "tok" + + monkeypatch.setattr( + connector_verify.alert_api_client, "get_api_access_token", fake_token + ) + monkeypatch.setattr( + connector_verify.alert_api_client, + "list_organizations", + lambda **kw: [{"id": 1, "name": "Ardeche"}, {"id": 2, "name": "Gard"}], + ) + + +async def test_test_endpoint_ok(authenticated_client, stub_alert_api): + response = await authenticated_client.post("/connectors/test", json=TEST_PAYLOAD) + assert response.status_code == 200 + body = response.json() + assert body["ok"] is True + assert body["organizations_total"] == 2 + assert body["error"] is None + + +async def test_test_endpoint_reports_auth_failure(authenticated_client, stub_alert_api): + response = await authenticated_client.post( + "/connectors/test", json={**TEST_PAYLOAD, "password": "bad"} + ) + assert response.status_code == 200 + body = response.json() + assert body["ok"] is False + assert "401" in body["error"] + assert "bad" not in body["error"] + + +async def test_test_endpoint_passes_scope_detail_through( + authenticated_client, stub_alert_api, monkeypatch +): + monkeypatch.setattr( + connector_verify.alert_api_client, + "list_organizations", + lambda **kw: {"detail": "Incompatible token scope."}, + ) + response = await authenticated_client.post("/connectors/test", json=TEST_PAYLOAD) + assert response.json()["ok"] is False + assert "Incompatible token scope." in response.json()["error"] + + +async def test_regular_user_cannot_test_credentials(regular_client, stub_alert_api): + response = await regular_client.post("/connectors/test", json=TEST_PAYLOAD) + assert response.status_code == 403 diff --git a/annotation_api/src/tests/endpoints/test_users.py b/annotation_api/src/tests/endpoints/test_users.py index aac2857f..98724929 100644 --- a/annotation_api/src/tests/endpoints/test_users.py +++ b/annotation_api/src/tests/endpoints/test_users.py @@ -429,7 +429,7 @@ async def test_delete_inactive_non_worker_user_allowed( {"username": "renamedworker"}, {"is_active": True}, {"is_superuser": True}, - {"is_active": False}, # same value as seeded — still rejected + {"is_active": False}, # opposite of seeded value — still rejected ], ) async def test_update_worker_user_forbidden( @@ -445,14 +445,20 @@ async def test_update_worker_user_forbidden( assert "Cannot modify the system worker user" in data["detail"] @pytest.mark.asyncio - async def test_update_worker_password_allowed( + async def test_update_worker_password_forbidden( self, authenticated_client: AsyncClient, worker_user: User ): - """Test the password endpoint is unaffected (worker cannot log in anyway).""" + """Test the worker's password cannot be set. The worker is active (see + app/main.py), so its random discarded password is the only thing + stopping anyone from logging in as it; setting a known password here + would let a superuser log in as an identity excluded from + annotator-facing lists by name, laundering their attribution.""" response = await authenticated_client.patch( f"/users/{worker_user.id}/password", json={"password": "newpassword123"} ) - assert response.status_code == 200 + assert response.status_code == 403 + data = response.json() + assert "Cannot modify the system worker user" in data["detail"] @pytest.mark.asyncio async def test_update_worker_user_empty_payload_allowed( diff --git a/annotation_api/src/tests/scripts/test_import_config.py b/annotation_api/src/tests/scripts/test_import_config.py new file mode 100644 index 00000000..6c6c5527 --- /dev/null +++ b/annotation_api/src/tests/scripts/test_import_config.py @@ -0,0 +1,306 @@ +"""The two new filters must cut work BEFORE the expensive per-sequence detection +fetches, which is the entire point of the short-circuit.""" + +from datetime import date + +from scripts.data_transfer.ingestion.alert_api import runner +from scripts.data_transfer.ingestion.alert_api.runner import ImportConfig, run_import +from scripts.data_transfer.ingestion.alert_api.sequence_fetching import ( + filter_sequences, +) + +CAMERA_ORG = {10: 1, 20: 2, 30: 3} +SEQUENCES = [ + {"id": 100, "camera_id": 10}, + {"id": 101, "camera_id": 20}, + {"id": 102, "camera_id": 30}, + {"id": 103, "camera_id": 10}, +] + + +def test_organization_filter_keeps_only_enabled_orgs(): + kept = filter_sequences( + SEQUENCES, camera_org=CAMERA_ORG, organization_ids={1, 3}, skip_ids=set() + ) + assert [s["id"] for s in kept] == [100, 102, 103] + + +def test_none_organization_filter_keeps_everything(): + kept = filter_sequences( + SEQUENCES, camera_org=CAMERA_ORG, organization_ids=None, skip_ids=set() + ) + assert len(kept) == 4 + + +def test_skip_ids_drop_already_imported_alerts(): + kept = filter_sequences( + SEQUENCES, camera_org=CAMERA_ORG, organization_ids=None, skip_ids={100, 103} + ) + assert [s["id"] for s in kept] == [101, 102] + + +def test_both_filters_compose(): + kept = filter_sequences( + SEQUENCES, camera_org=CAMERA_ORG, organization_ids={1}, skip_ids={100} + ) + assert [s["id"] for s in kept] == [103] + + +def test_sequence_with_unknown_camera_is_dropped_when_filtering_by_org(): + # A camera missing from the index cannot be attributed to an organization; + # importing it would silently ingest an org the operator never enabled. + kept = filter_sequences( + [{"id": 200, "camera_id": 999}], + camera_org=CAMERA_ORG, + organization_ids={1}, + skip_ids=set(), + ) + assert kept == [] + + +def test_sequence_with_unknown_camera_is_kept_when_not_filtering(): + kept = filter_sequences( + [{"id": 200, "camera_id": 999}], + camera_org=CAMERA_ORG, + organization_ids=None, + skip_ids=set(), + ) + assert [s["id"] for s in kept] == [200] + + +def test_import_config_defaults_match_cli_behaviour(): + config = ImportConfig( + alert_api_url="https://alertapi.pyronear.org", + login="u", + password="p", + admin_login="au", + admin_password="ap", + annotation_api_url="http://api:5050", + annotation_api_token="tok", + date_from=date(2026, 8, 5), + date_end=date(2026, 8, 5), + source_api="pyronear_french", + ) + assert config.organization_ids is None + assert config.skip_platform_alert_ids == frozenset() + assert config.max_sequences == 0 + assert config.risk_score == "extreme" + + +# --- run_import wiring ------------------------------------------------------- +# +# One alert sequence per organization is listed; org 2 is not enabled and alert +# 102 is already imported, so only alert 100 may reach the detection fetch. +# Object-splitting turns alert 100 into two lanes: one gets created, the other +# comes back as already existing. + +LISTED = [ + {"id": 100, "camera_id": 10}, + {"id": 101, "camera_id": 20}, + {"id": 102, "camera_id": 10}, +] +INDEXED_CAMERAS = { + 10: {"id": 10, "organization_id": 1}, + 20: {"id": 20, "organization_id": 2}, +} +SPLIT_RECORDS = [ + {"sequence_id": 100, "platform_alert_id": 100, "organization_id": 1}, + {"sequence_id": 1000100001, "platform_alert_id": 100, "organization_id": 1}, +] +POST_RESULT = { + "successful_sequences": 1, + "failed_sequences": 0, + "skipped_sequences": 1, + "total_sequences": 2, + "successful_detections": 1, + "failed_detections": 0, + "skipped_detections": 1, + "total_detections": 2, + "successful_sequence_ids": [500], + "sequence_results": [ + {"sequence_id": 500, "alert_api_sequence_id": 100, "failed_detections": 0}, + {"sequence_id": None, "alert_api_sequence_id": 1000100001, "skipped": True}, + ], +} + + +def _stub_pipeline(monkeypatch) -> dict: + """Replace every network-touching stage of run_import with a canned result. + + Returns what the stubs observed: the sequences the detection fetch was asked + for, and the tokens the posting and annotation stages were handed. + """ + captured: dict = {"fetched": [], "post_token": None, "annotate_tokens": []} + fetched = captured["fetched"] + annotate_tokens = captured["annotate_tokens"] + monkeypatch.setattr( + runner.alert_api_client, "get_api_access_token", lambda **kwargs: "token" + ) + monkeypatch.setattr( + runner, + "load_alert_api_metadata", + lambda **kwargs: (INDEXED_CAMERAS, {1: {"name": "org1"}, 2: {"name": "org2"}}), + ) + monkeypatch.setattr(runner, "list_sequences_within", lambda **kwargs: list(LISTED)) + + def fake_fetch_detections(*, sequences, **kwargs): + fetched.extend(sequences) + return list(SPLIT_RECORDS) + + monkeypatch.setattr(runner, "fetch_detections_for_sequences", fake_fetch_detections) + monkeypatch.setattr( + runner.object_split, + "split_all_records", + lambda records: ( + list(SPLIT_RECORDS), + { + "alert_api_sequences": 1, + "objects": 2, + "sibling_objects": 1, + "fallback_sequences": 0, + "cross_deduped_siblings": 0, + # Real split_all_records returns these too (object_split.py); + # the runner's summary line reads every key, so a stub that + # omits one raises KeyError inside run_import. + "same_frame_merges": 0, + "dropped_temporal_scores": 0, + }, + ), + ) + + def fake_post(*args, **kwargs): + captured["post_token"] = kwargs.get("auth_token") + return POST_RESULT + + monkeypatch.setattr(runner.shared, "post_records_to_annotation_api", fake_post) + # SPLIT_RECORDS carries no "detection_bboxes" (these tests aren't about + # boxless alerts), so stub the classifier out rather than let it KeyError. + monkeypatch.setattr( + runner.shared, "boxless_platform_alert_ids", lambda records: set() + ) + + def fake_annotate(*, seq_result, annotation_api_url, dry_run, auth_token): + annotate_tokens.append(auth_token) + return { + "sequence_id": seq_result["sequence_id"], + "annotation_created": True, + "annotation_id": 1, + "errors": [], + "final_stage": "ready_to_annotate", + } + + monkeypatch.setattr(runner, "annotate_split_sequence", fake_annotate) + return captured + + +def _config(**overrides) -> ImportConfig: + kwargs = dict( + alert_api_url="https://alertapi.pyronear.org", + login="u", + password="p", + admin_login="au", + admin_password="ap", + annotation_api_url="http://api:5050", + annotation_api_token="tok", + date_from=date(2026, 8, 5), + date_end=date(2026, 8, 5), + source_api="pyronear_french", + ) + kwargs.update(overrides) + return ImportConfig(**kwargs) + + +def test_run_import_filters_before_fetching_detections(monkeypatch): + captured = _stub_pipeline(monkeypatch) + + result = run_import( + _config(organization_ids={1}, skip_platform_alert_ids=frozenset({102})) + ) + + assert result.ok + # The whole point: the disabled org and the already-imported alert never + # reach the per-sequence detection fetch. + assert [s["id"] for s in captured["fetched"]] == [100] + + +def test_run_import_passes_the_configured_token_to_every_stage(monkeypatch): + # The worker self-mints a JWT precisely so no plaintext annotation-API + # password has to exist in its environment; a config token that never + # reaches the annotation API calls would silently defeat that. + captured = _stub_pipeline(monkeypatch) + + run_import(_config(annotation_api_token="worker-jwt", organization_ids={1})) + + assert captured["post_token"] == "worker-jwt" + assert captured["annotate_tokens"] == ["worker-jwt"] + + +def test_run_import_auto_skips_boxless_alerts_using_the_configured_token(monkeypatch): + # #333 auto-skip must reuse the already-resolved token (ImportConfig. + # annotation_api_token) rather than mint one from login/password: the + # worker has no ANNOTATOR_LOGIN/ANNOTATOR_PASSWORD in its environment, so + # a get_auth_token(login, password) call here would fail in that caller. + _stub_pipeline(monkeypatch) + monkeypatch.setattr( + runner.shared, "boxless_platform_alert_ids", lambda records: {100} + ) + skip_calls = [] + + def fake_skip_boxless_alerts(base_url, auth_token, source_api, platform_alert_ids): + skip_calls.append((base_url, auth_token, source_api, list(platform_alert_ids))) + return {"skipped": 1, "already_skipped": 0, "failed": 0} + + monkeypatch.setattr(runner.shared, "skip_boxless_alerts", fake_skip_boxless_alerts) + + result = run_import( + _config(annotation_api_token="worker-jwt", organization_ids={1}) + ) + + assert result.ok + # No login/password auth call was made: skip_boxless_alerts was reached + # with the configured token directly, and only once. + assert len(skip_calls) == 1 + base_url, auth_token, source_api, alert_ids = skip_calls[0] + assert auth_token == "worker-jwt" + assert alert_ids == [100] + + +def test_run_import_reports_per_organization_stats(monkeypatch): + _stub_pipeline(monkeypatch) + + result = run_import( + _config(organization_ids={1}, skip_platform_alert_ids=frozenset({102})) + ) + + org1 = result.per_organization[1] + assert org1.alerts_fetched == 2 # alerts 100 and 102 belong to org 1 + assert org1.alerts_skipped == 1 # alert 102 was filtered as already imported + assert org1.alerts_imported == 1 # alert 100 had a lane created + assert org1.alerts_failed == 0 + assert org1.lanes_created == 1 # its sibling lane already existed + + org2 = result.per_organization[2] + assert org2.alerts_fetched == 1 + assert org2.alerts_imported == 0 + assert org2.lanes_created == 0 + + +def test_run_import_counts_unreported_lanes_as_failures(monkeypatch): + # post_records_to_annotation_api only reports lanes it created or skipped, + # so a lane missing from sequence_results is one that failed. + _stub_pipeline(monkeypatch) + monkeypatch.setattr( + runner.shared, + "post_records_to_annotation_api", + lambda *args, **kwargs: { + **POST_RESULT, + "sequence_results": [POST_RESULT["sequence_results"][0]], + }, + ) + + result = run_import(_config(organization_ids={1})) + + org1 = result.per_organization[1] + assert org1.alerts_failed == 1 + assert org1.alerts_imported == 0 + assert org1.lanes_created == 1 diff --git a/annotation_api/src/tests/scripts/test_shared_posting.py b/annotation_api/src/tests/scripts/test_shared_posting.py index 481aadbc..ba77f8fe 100644 --- a/annotation_api/src/tests/scripts/test_shared_posting.py +++ b/annotation_api/src/tests/scripts/test_shared_posting.py @@ -141,6 +141,70 @@ def conflicting_sequence(url, token, data): assert result["successful_sequences"] == 0 +class TestSuppliedAuthToken: + """A caller-supplied token must be used verbatim, with no login round-trip. + + The worker self-mints its JWT so that no plaintext annotation-API password + has to exist in its environment; a token that got silently replaced by an + env-credential login would defeat that without failing anything. + """ + + def test_supplied_token_is_used_and_no_login_happens(self, monkeypatch): + def fail_login(*args, **kwargs): + raise AssertionError("must not log in when a token was supplied") + + monkeypatch.setattr(shared, "get_auth_token", fail_login) + + tokens = [] + monkeypatch.setattr( + shared, + "create_sequence", + lambda url, token, data: tokens.append(token) or {"id": 99}, + ) + monkeypatch.setattr( + shared, + "create_detection_from_bucket_key", + lambda url, token, detection_data, source_key: tokens.append(token) + or {"id": 501}, + ) + + records = [make_record(1, "2026-07-01T10:00:00", [BOX])] + result = shared.post_records_to_annotation_api( + "http://annotation.test", + records, + max_workers=1, + max_detection_workers=1, + auth_token="worker-jwt", + ) + assert result["successful_sequences"] == 1 + assert tokens == ["worker-jwt", "worker-jwt"] + + def test_without_a_token_it_still_logs_in(self, monkeypatch): + monkeypatch.setattr( + shared, "get_annotation_credentials", lambda url: ("u", "p") + ) + monkeypatch.setattr( + shared, "get_auth_token", lambda url, username, password: "env-token" + ) + tokens = [] + monkeypatch.setattr( + shared, + "create_sequence", + lambda url, token, data: tokens.append(token) or {"id": 99}, + ) + monkeypatch.setattr( + shared, + "create_detection_from_bucket_key", + lambda url, token, detection_data, source_key: {"id": 501}, + ) + + records = [make_record(1, "2026-07-01T10:00:00", [BOX])] + shared.post_records_to_annotation_api( + "http://annotation.test", records, max_workers=1, max_detection_workers=1 + ) + assert tokens == ["env-token"] + + class TestTransformSequenceData: def test_platform_alert_id_passed_through(self): record = make_record(1, "2026-07-01T10:00:00", [BOX]) diff --git a/annotation_api/src/tests/services/test_connector_import.py b/annotation_api/src/tests/services/test_connector_import.py new file mode 100644 index 00000000..1f9e629d --- /dev/null +++ b/annotation_api/src/tests/services/test_connector_import.py @@ -0,0 +1,254 @@ +"""The import job: window, organization filter, and coverage bookkeeping.""" + +from datetime import date, datetime + +import pytest +from cryptography.fernet import Fernet +from sqlmodel import select + +from app.core.config import settings +from app.models import ( + AlertApiConnector, + AlertApiConnectorOrganization, + AlertApiImportCoverage, + ImportCoverageStatus, + SourceApi, +) +from app.services import connector_import +from app.services.connector_import import import_connector +from app.services.secrets import encrypt_secret +from scripts.data_transfer.ingestion.alert_api.runner import ( + ImportResult, + OrganizationStats, +) + + +async def _fake_token(session): + return "worker-token" + + +@pytest.fixture +def secret_key(monkeypatch): + monkeypatch.setattr( + settings, "CONNECTOR_SECRET_KEY", Fernet.generate_key().decode() + ) + + +@pytest.fixture +async def connector(async_session, secret_key): + row = AlertApiConnector( + name="Test", + base_url="https://a.example", + source_api=SourceApi.PYRONEAR_FRENCH_API, + login="admin", + password_encrypted=encrypt_secret("pw"), + trailing_days=2, + ) + async_session.add(row) + await async_session.commit() + await async_session.refresh(row) + async_session.add( + AlertApiConnectorOrganization( + connector_id=row.id, + organization_id=1, + name="Ardeche", + is_enabled=True, + enabled_at=datetime(2026, 1, 1), + ) + ) + async_session.add( + AlertApiConnectorOrganization( + connector_id=row.id, organization_id=2, name="Aveyron", is_enabled=False + ) + ) + await async_session.commit() + return row + + +@pytest.fixture +def captured(monkeypatch): + """Capture every ImportConfig run_import is called with.""" + calls = [] + + def fake_run_import(config): + calls.append(config) + return ImportResult( + per_organization={ + 1: OrganizationStats( + alerts_fetched=5, + alerts_imported=3, + alerts_skipped=2, + lanes_created=4, + ) + } + ) + + monkeypatch.setattr(connector_import, "run_import", fake_run_import) + monkeypatch.setattr(connector_import, "mint_worker_token", _fake_token) + return calls + + +async def test_runs_one_import_per_day_in_the_window( + async_session, connector, captured +): + await import_connector(async_session, connector, today=date(2026, 8, 6)) + # trailing_days=2 -> Aug 4 and Aug 5. Today is never imported: the day is + # still in progress on the alert API. + assert [c.date_from for c in captured] == [date(2026, 8, 4), date(2026, 8, 5)] + assert all(c.date_from == c.date_end for c in captured) + + +async def test_passes_only_enabled_organizations(async_session, connector, captured): + await import_connector(async_session, connector, today=date(2026, 8, 6)) + assert captured[0].organization_ids == {1} + + +async def test_passes_the_minted_worker_token_to_every_days_config( + async_session, connector, captured +): + await import_connector(async_session, connector, today=date(2026, 8, 6)) + assert len(captured) == 2 + assert all(c.annotation_api_token == "worker-token" for c in captured) + + +async def test_writes_a_coverage_row_per_enabled_org_per_day( + async_session, connector, captured +): + await import_connector(async_session, connector, today=date(2026, 8, 6)) + rows = (await async_session.execute(select(AlertApiImportCoverage))).scalars().all() + assert {(r.organization_id, r.covered_date) for r in rows} == { + (1, date(2026, 8, 4)), + (1, date(2026, 8, 5)), + } + assert rows[0].status == ImportCoverageStatus.OK + assert rows[0].alerts_imported == 3 + assert rows[0].lanes_created == 4 + + +async def test_org_with_no_alerts_gets_an_ok_row_with_zeroes( + async_session, connector, monkeypatch +): + monkeypatch.setattr(connector_import, "mint_worker_token", _fake_token) + monkeypatch.setattr( + connector_import, "run_import", lambda config: ImportResult(per_organization={}) + ) + await import_connector(async_session, connector, today=date(2026, 8, 6)) + rows = (await async_session.execute(select(AlertApiImportCoverage))).scalars().all() + assert len(rows) == 2 + assert all(r.status == ImportCoverageStatus.OK for r in rows) + assert all(r.alerts_fetched == 0 for r in rows) + + +async def test_connector_failure_marks_every_enabled_org_failed( + async_session, connector, monkeypatch +): + monkeypatch.setattr(connector_import, "mint_worker_token", _fake_token) + + def boom(config): + raise RuntimeError("alert API down") + + monkeypatch.setattr(connector_import, "run_import", boom) + await import_connector(async_session, connector, today=date(2026, 8, 6)) + + rows = (await async_session.execute(select(AlertApiImportCoverage))).scalars().all() + assert len(rows) == 2 + assert all(r.status == ImportCoverageStatus.FAILED for r in rows) + assert all("alert API down" in (r.error or "") for r in rows) + + +async def test_partial_status_when_some_alerts_failed( + async_session, connector, monkeypatch +): + monkeypatch.setattr(connector_import, "mint_worker_token", _fake_token) + monkeypatch.setattr( + connector_import, + "run_import", + lambda config: ImportResult( + per_organization={ + 1: OrganizationStats( + alerts_fetched=5, alerts_imported=3, alerts_failed=2 + ) + } + ), + ) + await import_connector(async_session, connector, today=date(2026, 8, 6)) + rows = (await async_session.execute(select(AlertApiImportCoverage))).scalars().all() + assert all(r.status == ImportCoverageStatus.PARTIAL for r in rows) + + +async def test_rerunning_the_same_day_updates_rather_than_duplicates( + async_session, connector, captured +): + await import_connector(async_session, connector, today=date(2026, 8, 6)) + await import_connector(async_session, connector, today=date(2026, 8, 6)) + rows = (await async_session.execute(select(AlertApiImportCoverage))).scalars().all() + assert len(rows) == 2 + + +async def test_missing_secret_key_skips_without_writing_coverage( + async_session, connector, monkeypatch +): + monkeypatch.setattr(settings, "CONNECTOR_SECRET_KEY", "") + monkeypatch.setattr(connector_import, "mint_worker_token", _fake_token) + await import_connector(async_session, connector, today=date(2026, 8, 6)) + rows = (await async_session.execute(select(AlertApiImportCoverage))).scalars().all() + assert rows == [] + + +async def test_connector_with_no_enabled_orgs_does_nothing( + async_session, connector, captured +): + orgs = ( + (await async_session.execute(select(AlertApiConnectorOrganization))) + .scalars() + .all() + ) + for org in orgs: + org.is_enabled = False + async_session.add(org) + await async_session.commit() + + await import_connector(async_session, connector, today=date(2026, 8, 6)) + assert captured == [] + + +async def test_missing_worker_token_skips_without_importing( + async_session, connector, monkeypatch +): + """The worker user not existing yet (e.g. a cold-boot race with the API's + seeding) must stop before any alert-API call — never fall back to a + plaintext credential.""" + calls = [] + + async def no_token(session): + return None + + def fake_run_import(config): + calls.append(config) + return ImportResult(per_organization={}) + + monkeypatch.setattr(connector_import, "mint_worker_token", no_token) + monkeypatch.setattr(connector_import, "run_import", fake_run_import) + + await import_connector(async_session, connector, today=date(2026, 8, 6)) + + assert calls == [] + rows = (await async_session.execute(select(AlertApiImportCoverage))).scalars().all() + assert rows == [] + + +async def test_db_failure_mid_loop_does_not_propagate( + async_session, connector, captured, monkeypatch +): + """A DB-layer failure (here: the coverage commit) anywhere in the + function — not just around run_import — must be swallowed, not raised.""" + + async def boom_commit(): + raise RuntimeError("db down") + + monkeypatch.setattr(async_session, "commit", boom_commit) + + await import_connector(async_session, connector, today=date(2026, 8, 6)) + + # The first day's run_import call happened before the commit failed. + assert len(captured) == 1 diff --git a/annotation_api/src/tests/services/test_connector_verify.py b/annotation_api/src/tests/services/test_connector_verify.py new file mode 100644 index 00000000..09cbf28e --- /dev/null +++ b/annotation_api/src/tests/services/test_connector_verify.py @@ -0,0 +1,311 @@ +"""Verify: authenticate, discover organizations idempotently, and probe whether +the credential actually sees more than one organization's sequences.""" + +import time +from datetime import date, datetime + +import pytest +from cryptography.fernet import Fernet +from sqlmodel import select + +from app.core.config import settings +from app.models import AlertApiConnector, AlertApiConnectorOrganization, SourceApi +from app.services import connector_verify +from app.services.connector_verify import check_connector_credentials, verify_connector +from app.services.secrets import encrypt_secret + +ORGS = [ + {"id": 1, "name": "Ardeche"}, + {"id": 2, "name": "Aveyron"}, + {"id": 3, "name": "Gard"}, +] +CAMERAS = [ + {"id": 10, "name": "cam-a", "organization_id": 1}, + {"id": 20, "name": "cam-b", "organization_id": 2}, +] +SEQUENCES = [ + {"id": 100, "camera_id": 10}, + {"id": 101, "camera_id": 20}, + {"id": 102, "camera_id": 10}, +] + + +@pytest.fixture +def secret_key(monkeypatch): + monkeypatch.setattr( + settings, "CONNECTOR_SECRET_KEY", Fernet.generate_key().decode() + ) + + +@pytest.fixture +def alert_api(monkeypatch): + """Stub the alert API client at the seam connector_verify imports it from.""" + + def fake_token(api_endpoint, username, password): + if password != "good": + raise RuntimeError("401 Unauthorized") + return "tok" + + monkeypatch.setattr( + connector_verify.alert_api_client, "get_api_access_token", fake_token + ) + monkeypatch.setattr( + connector_verify.alert_api_client, "list_organizations", lambda **kw: ORGS + ) + monkeypatch.setattr( + connector_verify.alert_api_client, "list_cameras", lambda **kw: CAMERAS + ) + monkeypatch.setattr( + connector_verify.alert_api_client, + "list_sequences_for_date", + lambda **kw: SEQUENCES, + ) + + +async def _connector(session, password="good"): + connector = AlertApiConnector( + name="Test", + base_url="https://a.example", + source_api=SourceApi.PYRONEAR_FRENCH_API, + login="admin", + password_encrypted=encrypt_secret(password), + ) + session.add(connector) + await session.commit() + await session.refresh(connector) + return connector + + +async def test_discovers_and_persists_organizations( + async_session, alert_api, secret_key +): + connector = await _connector(async_session) + result = await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + assert result.ok is True + assert {org.organization_id for org in result.organizations} == {1, 2, 3} + rows = ( + (await async_session.execute(select(AlertApiConnectorOrganization))) + .scalars() + .all() + ) + assert len(rows) == 3 + assert all(row.is_enabled is False for row in rows) + + +async def test_rediscovery_is_idempotent_and_preserves_enabled( + async_session, alert_api, secret_key +): + connector = await _connector(async_session) + await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + row = ( + ( + await async_session.execute( + select(AlertApiConnectorOrganization).where( + AlertApiConnectorOrganization.organization_id == 2 + ) + ) + ) + .scalars() + .one() + ) + row.is_enabled = True + row.enabled_at = datetime(2026, 8, 1) + async_session.add(row) + await async_session.commit() + + await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + rows = ( + (await async_session.execute(select(AlertApiConnectorOrganization))) + .scalars() + .all() + ) + assert len(rows) == 3, "re-verifying must not duplicate organizations" + assert [r.is_enabled for r in rows if r.organization_id == 2] == [True] + + +async def test_reports_organizations_seen_in_sample( + async_session, alert_api, secret_key +): + connector = await _connector(async_session) + result = await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + # Sequences came from cameras in organizations 1 and 2, out of 3 known. + assert result.organizations_seen_in_sample == 2 + assert result.organizations_total == 3 + assert result.sample_date == date(2026, 8, 5) + + +async def test_bad_credentials_record_error_and_do_not_raise( + async_session, alert_api, secret_key +): + connector = await _connector(async_session, password="bad") + result = await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + assert result.ok is False + assert result.error + # Refresh explicitly: verify_connector commits, and reading an expired + # attribute would trigger a sync lazy load, which async SQLAlchemy forbids + # (MissingGreenlet). + await async_session.refresh(connector) + assert connector.last_verify_error + assert connector.last_verified_at is None + + +async def test_success_clears_previous_error(async_session, alert_api, secret_key): + connector = await _connector(async_session) + connector.last_verify_error = "stale failure" + async_session.add(connector) + await async_session.commit() + + result = await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + assert result.ok is True + await async_session.refresh(connector) + assert connector.last_verify_error is None + assert connector.last_verified_at is not None + + +async def test_error_message_never_contains_the_password( + async_session, alert_api, secret_key +): + connector = await _connector(async_session, password="bad") + result = await verify_connector(async_session, connector, today=date(2026, 8, 6)) + assert "bad" not in (result.error or "") + + +async def test_malformed_probe_response_does_not_raise( + async_session, alert_api, secret_key, monkeypatch +): + """`api_get` only raises on unparsable JSON, so a non-2xx response with a + valid JSON error body (e.g. an expired-token 401) comes back as a dict + where a list is expected. That must fail like any other verify error, not + raise a TypeError out of the org-upsert loop.""" + monkeypatch.setattr( + connector_verify.alert_api_client, + "list_organizations", + lambda **kw: {"detail": "Not authenticated"}, + ) + connector = await _connector(async_session) + result = await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + assert result.ok is False + assert result.error + await async_session.refresh(connector) + assert connector.last_verify_error + assert connector.last_verified_at is None + + +async def test_insufficient_scope_detail_reaches_the_operator( + async_session, alert_api, secret_key, monkeypatch +): + """Connectors need an admin-scoped alert-API credential, and nothing in the + UI says so. A non-admin account authenticates fine and then gets + `{"detail": "Incompatible token scope."}` from `/organizations/` (verified + against the real alert API, 2026-08-07) — so that detail is the only signal + telling the operator which credential to swap in. Reporting a bare + "unexpected response shape" throws it away.""" + monkeypatch.setattr( + connector_verify.alert_api_client, + "list_organizations", + lambda **kw: {"detail": "Incompatible token scope."}, + ) + connector = await _connector(async_session) + result = await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + assert result.ok is False + assert "Incompatible token scope." in (result.error or "") + + +async def test_probe_timeout_records_error_and_does_not_raise( + async_session, alert_api, secret_key, monkeypatch +): + """A host that black-holes packets must not hang verify forever: the probe + is wrapped in asyncio.wait_for, and the resulting TimeoutError must be + caught by the same `except Exception` as any other verify failure rather + than propagating out of verify_connector.""" + monkeypatch.setattr(connector_verify, "_PROBE_TIMEOUT_SECONDS", 0.05) + + def slow(**kw): + time.sleep(0.2) + return CAMERAS + + monkeypatch.setattr(connector_verify.alert_api_client, "list_cameras", slow) + connector = await _connector(async_session) + result = await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + assert result.ok is False + assert "Timeout" in (result.error or "") + await async_session.refresh(connector) + assert connector.last_verify_error + assert connector.last_verified_at is None + + +# --- check_connector_credentials: the stateless pre-save credential check --- + + +async def test_credentials_ok_reports_organization_count(alert_api): + result = await check_connector_credentials("https://a.example", "admin", "good") + assert result.ok is True + assert result.error is None + assert result.organizations_total == 3 + + +async def test_credentials_bad_password_reports_error(alert_api): + result = await check_connector_credentials("https://a.example", "admin", "bad") + assert result.ok is False + assert "401" in (result.error or "") + + +async def test_credentials_org_scoped_account_reports_scope_detail( + alert_api, monkeypatch +): + """The failure mode operators actually hit (verified against production + 2026-08-07): a non-admin credential authenticates, then /organizations/ + answers {"detail": "Incompatible token scope."}. That detail must reach + the operator.""" + monkeypatch.setattr( + connector_verify.alert_api_client, + "list_organizations", + lambda **kw: {"detail": "Incompatible token scope."}, + ) + result = await check_connector_credentials("https://a.example", "admin", "good") + assert result.ok is False + assert "Incompatible token scope." in (result.error or "") + + +async def test_credentials_never_echo_the_password(alert_api): + result = await check_connector_credentials("https://a.example", "admin", "bad") + assert "bad" not in (result.error or "") + + +async def test_credentials_timeout_is_bounded(alert_api, monkeypatch): + monkeypatch.setattr(connector_verify, "_TEST_TIMEOUT_SECONDS", 0.05) + + def slow(**kw): + time.sleep(0.2) + return ORGS + + monkeypatch.setattr(connector_verify.alert_api_client, "list_organizations", slow) + result = await check_connector_credentials("https://a.example", "admin", "good") + assert result.ok is False + assert "Timeout" in (result.error or "") + + +async def test_connection_error_during_probe_does_not_raise( + async_session, alert_api, secret_key, monkeypatch +): + def boom(**kw): + raise ConnectionError("network unreachable") + + monkeypatch.setattr(connector_verify.alert_api_client, "list_cameras", boom) + connector = await _connector(async_session) + result = await verify_connector(async_session, connector, today=date(2026, 8, 6)) + + assert result.ok is False + assert result.error + await async_session.refresh(connector) + assert connector.last_verify_error + assert connector.last_verified_at is None diff --git a/annotation_api/src/tests/services/test_secrets.py b/annotation_api/src/tests/services/test_secrets.py new file mode 100644 index 00000000..fea454d4 --- /dev/null +++ b/annotation_api/src/tests/services/test_secrets.py @@ -0,0 +1,54 @@ +"""Fernet round-trip for connector credentials, plus the failure modes that must +produce an actionable message rather than a stack trace.""" + +import pytest +from cryptography.fernet import Fernet + +from app.core.config import settings +from app.services.secrets import ( + SecretKeyMissingError, + decrypt_secret, + encrypt_secret, +) + + +@pytest.fixture +def secret_key(monkeypatch) -> str: + key = Fernet.generate_key().decode() + monkeypatch.setattr(settings, "CONNECTOR_SECRET_KEY", key) + return key + + +def test_round_trip(secret_key): + assert decrypt_secret(encrypt_secret("hunter2")) == "hunter2" + + +def test_ciphertext_does_not_contain_plaintext(secret_key): + assert "hunter2" not in encrypt_secret("hunter2") + + +def test_encrypt_twice_gives_different_tokens(secret_key): + # Fernet embeds a timestamp and IV, so identical plaintext must not produce + # identical ciphertext. + assert encrypt_secret("hunter2") != encrypt_secret("hunter2") + + +def test_missing_key_raises_named_error(monkeypatch): + monkeypatch.setattr(settings, "CONNECTOR_SECRET_KEY", "") + with pytest.raises(SecretKeyMissingError, match="CONNECTOR_SECRET_KEY"): + encrypt_secret("hunter2") + + +def test_malformed_key_raises_named_error(monkeypatch): + monkeypatch.setattr(settings, "CONNECTOR_SECRET_KEY", "not-a-fernet-key") + with pytest.raises(SecretKeyMissingError, match="CONNECTOR_SECRET_KEY"): + encrypt_secret("hunter2") + + +def test_decrypt_with_wrong_key_raises_named_error(monkeypatch, secret_key): + token = encrypt_secret("hunter2") + monkeypatch.setattr( + settings, "CONNECTOR_SECRET_KEY", Fernet.generate_key().decode() + ) + with pytest.raises(SecretKeyMissingError): + decrypt_secret(token) diff --git a/annotation_api/src/tests/services/test_worker_auth.py b/annotation_api/src/tests/services/test_worker_auth.py new file mode 100644 index 00000000..7fd99a43 --- /dev/null +++ b/annotation_api/src/tests/services/test_worker_auth.py @@ -0,0 +1,92 @@ +"""The worker authenticates to its own API with a self-minted JWT — no password +in its environment, and no dependency on the API having seeded users first. + +The third test is a PINNING test. app.api.dependencies.get_current_user is an +alias for get_current_active_user (see that module), so the worker user MUST be +seeded active (app/main.py) for a minted token to be accepted by the sequence and +detection endpoints — login stays blocked separately, by the random discarded +password (see test_worker_user_cannot_login in test_user_seeding.py). This test +runs the real seed_default_users startup routine — not a fixture that merely +mimics it — and mints a token for the worker it seeds, so it fails end to end if +either the seed is ever flipped back to inactive or the sequence endpoint's auth +dependency changes shape, instead of nightly imports silently breaking. +""" + +from datetime import UTC, datetime, timedelta + +from httpx import ASGITransport, AsyncClient + +from app.auth.dependencies import verify_token +from app.core.config import settings +from app.db import get_session +from app.main import app, seed_default_users +from app.services.worker_auth import mint_worker_token + +_now = datetime.now(UTC) + +# Copied from a known-good payload in src/tests/endpoints/test_sequence.py +# (test_create_sequence, ~line 20-33) — SequenceCreate is strict and the +# endpoint parses these as Form fields, not a JSON body. +SEQUENCE_PAYLOAD = { + "source_api": "pyronear_french", + "alert_api_id": "100", + "camera_name": "test_cam", + "camera_id": "1", + "organisation_name": "test_org", + "organisation_id": "1", + "is_wildfire_alertapi": "wildfire_smoke", + "azimuth": "90", + "lat": "0.0", + "lon": "0.0", + "created_at": (_now - timedelta(days=1)).isoformat(), + "recorded_at": (_now - timedelta(days=1)).isoformat(), + "last_seen_at": _now.isoformat(), +} + + +async def test_returns_none_when_worker_user_absent(async_session): + assert await mint_worker_token(async_session) is None + + +async def test_mints_a_token_for_the_worker_user(async_session, worker_user): + token = await mint_worker_token(async_session) + assert token is not None + payload = verify_token(token) + assert payload is not None + assert payload.user_id == worker_user.id + assert payload.username == worker_user.username + + +async def test_worker_token_can_create_a_sequence(async_session): + """PINNING TEST — see the module docstring. Do not delete this to make a + refactor pass; if it fails, the worker needs a different identity. + + Deliberately does not use the worker_user fixture: that fixture builds its + own User row and would stay green even if the real startup seed in + app.main.seed_default_users regressed back to is_active=False. Running the + actual seed routine is what makes this test end to end over the real seed. + """ + await seed_default_users(async_session) + token = await mint_worker_token(async_session) + + async def get_test_session(): + yield async_session + + app.dependency_overrides[get_session] = get_test_session + try: + async with AsyncClient( + transport=ASGITransport(app=app), + base_url=f"http://api.localhost:8050{settings.API_V1_STR}", + headers={"Authorization": f"Bearer {token}"}, + follow_redirects=True, + timeout=5, + ) as client: + response = await client.post("/sequences/", data=SEQUENCE_PAYLOAD) + finally: + app.dependency_overrides.clear() + + assert response.status_code in (200, 201), ( + f"Worker-minted token was rejected ({response.status_code}): {response.text}. " + "Either the worker seed was flipped back to inactive, or the sequence " + "endpoint's auth dependency changed — see this module's docstring." + ) diff --git a/annotation_api/src/tests/test_user_seeding.py b/annotation_api/src/tests/test_user_seeding.py index 7316f4cc..effcecba 100644 --- a/annotation_api/src/tests/test_user_seeding.py +++ b/annotation_api/src/tests/test_user_seeding.py @@ -1,4 +1,4 @@ -"""Tests for startup user seeding (admin + login-disabled worker user).""" +"""Tests for startup user seeding (admin + password-disabled worker user).""" import logging @@ -18,7 +18,11 @@ async def test_seed_creates_login_disabled_worker_user( await seed_default_users(async_session) worker = await UserCRUD(async_session).get_by_username(settings.WORKER_USERNAME) assert worker is not None - assert worker.is_active is False + # Must be active: app.api.dependencies.get_current_user is an alias for + # get_current_active_user, so an inactive worker could not call the + # endpoints it posts to. Login is blocked by the discarded random + # password instead — see test_worker_user_cannot_login below. + assert worker.is_active is True assert worker.is_superuser is False @@ -48,10 +52,16 @@ async def test_worker_user_cannot_login( async_session: AsyncSession, ): await seed_default_users(async_session) + worker = await UserCRUD(async_session).get_by_username(settings.WORKER_USERNAME) + # The worker is active (required so it can call the API — see + # test_seed_creates_login_disabled_worker_user). This test is therefore + # the *only* guard that login stays blocked: it must fail on the + # discarded random password alone, not on an is_active check. + assert worker is not None and worker.is_active is True resp = await async_client.post( "/auth/login", json={"username": settings.WORKER_USERNAME, "password": "anything"}, ) - # Wrong password -> 401 before the is_active check even runs; the - # password is random-and-discarded so no password can ever be right. + # Wrong password -> 401; the password is random-and-discarded so no + # password can ever be right. assert resp.status_code == 401 diff --git a/annotation_api/src/tests/test_worker_periodic.py b/annotation_api/src/tests/test_worker_periodic.py index be343794..e4b5faf5 100644 --- a/annotation_api/src/tests/test_worker_periodic.py +++ b/annotation_api/src/tests/test_worker_periodic.py @@ -16,3 +16,18 @@ def test_assign_sequence_groups_periodic_cron(): ] assert len(entries) == 1 assert entries[0].cron == "*/5 * * * *" + + +def test_connector_import_tasks_registered(): + assert "schedule_connector_imports" in procrastinate_app.tasks + assert "run_connector_import" in procrastinate_app.tasks + + +def test_schedule_connector_imports_runs_daily(): + entries = [ + pt + for pt in procrastinate_app.periodic_registry.periodic_tasks.values() + if pt.task.name == "schedule_connector_imports" + ] + assert len(entries) == 1 + assert entries[0].cron == "0 3 * * *" diff --git a/annotation_api/uv.lock b/annotation_api/uv.lock index bd252855..3880c02d 100644 --- a/annotation_api/uv.lock +++ b/annotation_api/uv.lock @@ -66,6 +66,7 @@ dependencies = [ { name = "asyncpg" }, { name = "bcrypt" }, { name = "boto3" }, + { name = "cryptography" }, { name = "fastapi" }, { name = "fastapi-pagination" }, { name = "httpx" }, @@ -80,8 +81,10 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "pyjwt" }, + { name = "python-dotenv" }, { name = "python-magic" }, { name = "python-multipart" }, + { name = "pyyaml" }, { name = "requests" }, { name = "rich" }, { name = "sentry-sdk", extra = ["fastapi"] }, @@ -100,7 +103,6 @@ dev = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-pretty" }, - { name = "pyyaml" }, { name = "requests-mock" }, { name = "ruff" }, { name = "sqlalchemy-stubs" }, @@ -115,6 +117,7 @@ requires-dist = [ { name = "asyncpg", specifier = ">=0.25.0,<1.0.0" }, { name = "bcrypt", specifier = ">=3.2.0,<4.0.0" }, { name = "boto3", specifier = ">=1.26.0" }, + { name = "cryptography", specifier = ">=42.0.0" }, { name = "fastapi", specifier = ">=0.109.1,<1.0.0" }, { name = "fastapi-pagination", specifier = ">=0.13.3" }, { name = "httpx", specifier = ">=0.24.0" }, @@ -129,8 +132,10 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.0.0,<3.0.0" }, { name = "pyjwt", specifier = ">=2.8.0" }, + { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "python-magic", specifier = ">=0.4.17" }, { name = "python-multipart", specifier = "==0.0.7" }, + { name = "pyyaml", specifier = ">=6.0" }, { name = "requests", specifier = ">=2.32.0" }, { name = "rich", specifier = ">=13.0.0" }, { name = "sentry-sdk", extras = ["fastapi"], specifier = ">=2.8.0" }, @@ -149,7 +154,6 @@ dev = [ { name = "pytest-asyncio", specifier = ">=0.17.0,<1.0.0" }, { name = "pytest-cov", specifier = ">=4.0.0" }, { name = "pytest-pretty", specifier = ">=1.0.0" }, - { name = "pyyaml", specifier = ">=6.0" }, { name = "requests-mock", specifier = ">=1.11.0" }, { name = "ruff", specifier = ">=0.7.1" }, { name = "sqlalchemy-stubs", specifier = ">=0.4" }, @@ -236,7 +240,8 @@ name = "bcrypt" version = "3.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi" }, + { name = "cffi", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and platform_python_implementation == 'PyPy'" }, + { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or platform_python_implementation != 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e8/36/edc85ab295ceff724506252b774155eff8a238f13730c8b13badd33ef866/bcrypt-3.2.2.tar.gz", hash = "sha256:433c410c2177057705da2a9f2cd01dd157493b2a7ac14c8593a16b3dab6b6bfb", size = 42455, upload-time = "2022-05-01T17:58:52.348Z" } wheels = [ @@ -293,6 +298,14 @@ wheels = [ name = "cffi" version = "1.17.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'darwin'", + "python_full_version >= '3.13' and platform_machine == 'aarch64' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_machine == 'aarch64' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "(python_full_version >= '3.13' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and sys_platform == 'linux') or (python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'darwin' and sys_platform != 'linux')", + "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and platform_python_implementation == 'PyPy' and sys_platform == 'linux') or (python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'darwin' and sys_platform != 'linux')", +] dependencies = [ { name = "pycparser" }, ] @@ -334,6 +347,118 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, ] +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'darwin'", + "python_full_version >= '3.14' and platform_machine == 'aarch64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_machine == 'aarch64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and platform_machine == 'aarch64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "(python_full_version >= '3.14' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux') or (python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'darwin' and sys_platform != 'linux')", + "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux') or (python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'darwin' and sys_platform != 'linux')", + "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux') or (python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.12' and sys_platform == 'darwin'", + "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux')", +] +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/d2/16d99a0c4948febc0ebd133a13b2f688ff7f8cb04da971e1128872ce0c03/cffi-2.1.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12", size = 183838, upload-time = "2026-08-03T21:19:29.637Z" }, + { url = "https://files.pythonhosted.org/packages/cd/95/31b535a9f0220ae9f357de4a08d57ce89cb417653c2fd9f075f50822a388/cffi-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1", size = 184168, upload-time = "2026-08-03T21:19:30.764Z" }, + { url = "https://files.pythonhosted.org/packages/ad/5a/4707a0dc1f203f5dde5a907b0d4e3c25d71120241048bd5bc6f1bb9d4e71/cffi-2.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0", size = 211805, upload-time = "2026-08-03T21:19:31.867Z" }, + { url = "https://files.pythonhosted.org/packages/ad/66/c19feabb28485b6e0bbaaafa90837a1ef5d302e90f2178bd33f17a49879b/cffi-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813", size = 218716, upload-time = "2026-08-03T21:19:32.896Z" }, + { url = "https://files.pythonhosted.org/packages/a7/92/500760486c8baab49a7a8a58ba7fc3355ec3974b454b8a09e528efde9e1d/cffi-2.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990", size = 205569, upload-time = "2026-08-03T21:19:34.142Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a7/a67c733254d6e7373f7822f8082d8d6beade791e0cf12a7611f376fa61c7/cffi-2.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af", size = 204907, upload-time = "2026-08-03T21:19:35.174Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a4/4399daaf8f7dfee9d7c3327fdb0426ee041cc63edc358b93911ceb2bfc7a/cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632", size = 217807, upload-time = "2026-08-03T21:19:36.286Z" }, + { url = "https://files.pythonhosted.org/packages/28/f7/dabe6da2466ecbd82dc62e7342dc6b1065dad990c06f00f0ede9ebf2a0ed/cffi-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd", size = 221252, upload-time = "2026-08-03T21:19:37.416Z" }, + { url = "https://files.pythonhosted.org/packages/ce/87/616202d8e51342c07d2534c510111c4cc37201775ce8f60802c9335d1edd/cffi-2.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a", size = 214214, upload-time = "2026-08-03T21:19:38.507Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c6/ab025d75d2c26c19b087c0124e75ee31cb65032f4fe345d356d8c507ab97/cffi-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa", size = 219408, upload-time = "2026-08-03T21:19:39.809Z" }, + { url = "https://files.pythonhosted.org/packages/db/e2/7e8109f65445bdc673a7b54f02c677de462db75674220fd1335efc8eb598/cffi-2.1.1-cp311-cp311-win32.whl", hash = "sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3", size = 174470, upload-time = "2026-08-03T21:19:41.246Z" }, + { url = "https://files.pythonhosted.org/packages/73/c0/77ba02423c2f7d7091143c45cd49e0e6575c4c1967394bb542bd923a9b74/cffi-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0", size = 185096, upload-time = "2026-08-03T21:19:42.615Z" }, + { url = "https://files.pythonhosted.org/packages/7c/47/9f1f85f9672ceda4984dc6c4f8824e8558992a2972c3d3c81fb8eb28d4ba/cffi-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455", size = 179941, upload-time = "2026-08-03T21:19:43.747Z" }, + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, + { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, + { url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" }, + { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, + { url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" }, + { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" }, + { url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" }, + { url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + [[package]] name = "cfgv" version = "3.4.0" @@ -516,6 +641,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cd/ba/d678e5bd329646ca51d3c92addbc77804e86d21f4b6b6a027218e6abb010/croniter-6.2.4-py3-none-any.whl", hash = "sha256:8ef3d544107a5c05a150a2d78f8bf5a8eb9c5c4d93405a736b824109574e3f4d", size = 46677, upload-time = "2026-07-10T09:52:58.425Z" }, ] +[[package]] +name = "cryptography" +version = "50.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", version = "2.1.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/5c/59086b4aac5e879d38ddbcf74e4be7ade89cebc3eb199a55da998c3bb46a/cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03", size = 4001252, upload-time = "2026-07-31T14:23:33.331Z" }, + { url = "https://files.pythonhosted.org/packages/57/ef/8f2df13c7216bcad3e1c74e07f6e193d93e998e114f524a53877c9af27ad/cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645", size = 4719554, upload-time = "2026-07-31T14:23:35.611Z" }, + { url = "https://files.pythonhosted.org/packages/d9/41/029086c34d91052fc3b88bcc8056f709a7c915c7a23b235a54eb800b1c97/cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7", size = 4702130, upload-time = "2026-07-31T14:23:37.635Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ff/b6ce0954962e7f7b969f850a883744197bb3910bdfd7b6da162eab7d9f68/cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3", size = 4725244, upload-time = "2026-07-31T14:23:39.471Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/63a1027cb7fec360a182208e1b7767d5aa1fe57be3d6aa856e69a321edc0/cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f", size = 5342265, upload-time = "2026-07-31T14:23:41.286Z" }, + { url = "https://files.pythonhosted.org/packages/6b/72/a1116d683a6d7ece94590013882515de087edf9ef0e6292aae615a44df73/cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae", size = 4734609, upload-time = "2026-07-31T14:23:43.139Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/36a9c479bbe49acea2636c7fd3360d20f7b7e079c300352011c44850b181/cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a", size = 4356517, upload-time = "2026-07-31T14:23:44.939Z" }, + { url = "https://files.pythonhosted.org/packages/32/98/8a151d64367204cbc63ec65d37502f1d9c53cf4bfc6ec3c532614dbec60d/cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987", size = 4724529, upload-time = "2026-07-31T14:23:46.93Z" }, + { url = "https://files.pythonhosted.org/packages/22/f6/ec13b470172126464a86bf54d2294a46d29837fc51ba3e45d4047946fb5e/cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169", size = 5299852, upload-time = "2026-07-31T14:23:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/da/3a/f05e32c99d440c9bb891ea0e36c9091891e36be5a9a87ab2ee6ea20729f6/cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f", size = 4734462, upload-time = "2026-07-31T14:23:50.861Z" }, + { url = "https://files.pythonhosted.org/packages/ca/dc/bd72b26be8953f80625f63151efd38eee71c76ca6cf591c08ff34615a79e/cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105", size = 4852708, upload-time = "2026-07-31T14:23:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/27/20/c930314a2ab476d15dec966ec87e2e9637bb02b06106b12c0396c57bb603/cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef", size = 5004179, upload-time = "2026-07-31T14:23:54.887Z" }, + { url = "https://files.pythonhosted.org/packages/32/2e/c9db68a0c4bfa28e310707527c0ee3a2bd254104d2e02e68f368e197aa4c/cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30", size = 3840395, upload-time = "2026-07-31T14:23:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fb/951032a3bf22a5697c83183fb6294a4843772947a70e616c57b3ff5f522e/cryptography-50.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41", size = 3989258, upload-time = "2026-07-31T14:23:58.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/67/91eb047e69c5e845f2f14b8a2e4a1aab0f283cb885531e9e22c8adb176bc/cryptography-50.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc", size = 4700648, upload-time = "2026-07-31T14:24:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/30/82/85f0f7425c856b9f96459411eb12e74ef72df9caf6f8f15bf23a33ff131f/cryptography-50.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f", size = 4682442, upload-time = "2026-07-31T14:24:02.538Z" }, + { url = "https://files.pythonhosted.org/packages/1a/28/b555a365adff1cca2fbe7b9e487d68a40de6bc67ff2cb587473eb43de0e7/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3", size = 4707596, upload-time = "2026-07-31T14:24:04.394Z" }, + { url = "https://files.pythonhosted.org/packages/72/d8/f52538140cc719df62a01cf87d1c7142318d235817109d6f4054d7c352d6/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533", size = 5314552, upload-time = "2026-07-31T14:24:06.31Z" }, + { url = "https://files.pythonhosted.org/packages/38/14/6120e5bd7c5aa022ad15424ba4d5c5269d0d9448ed4d55e492ea91e3c1c4/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037", size = 4717113, upload-time = "2026-07-31T14:24:08.349Z" }, + { url = "https://files.pythonhosted.org/packages/fa/71/190bf38c3ee2e0f8efc9860ae100c9df4169742eef274b91e7aa1cb133b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f", size = 4338580, upload-time = "2026-07-31T14:24:10.227Z" }, + { url = "https://files.pythonhosted.org/packages/3a/63/504ccfbbe61fd8aa983f7f146399cdf034c72c2fc55f5b2dfdcdcdb20c99/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11", size = 4707038, upload-time = "2026-07-31T14:24:12.169Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/2cf79bbfc4d12ca106437a6e170d6aaa01a373e93093118aaaef0e801bd4/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d", size = 5273110, upload-time = "2026-07-31T14:24:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e5/45/8aae2972c520145377ea3559a605a899bebe227bf070b33cdb445929a9b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c", size = 4716439, upload-time = "2026-07-31T14:24:16.415Z" }, + { url = "https://files.pythonhosted.org/packages/7b/20/4fe50b619a48c2525cc46e2dbc1ac490708d704be5d467bdaac6dc955682/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c", size = 4837383, upload-time = "2026-07-31T14:24:18.553Z" }, + { url = "https://files.pythonhosted.org/packages/92/91/3a31366e183343d3703f8995c095f5734676bd6938118047e50fcf279eb4/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95", size = 4985772, upload-time = "2026-07-31T14:24:20.385Z" }, + { url = "https://files.pythonhosted.org/packages/74/9a/02ffe35b2853d121689871eb5dce862092562b3a1ed5cc98f1aaed441506/cryptography-50.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269", size = 3816291, upload-time = "2026-07-31T14:24:22.125Z" }, + { url = "https://files.pythonhosted.org/packages/03/37/73d005be173aff344af30e9fd2a576575cb2391a7101d9cd3842e1fa8cce/cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07", size = 4036009, upload-time = "2026-07-31T14:24:24.122Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c6/7a6202a534e32103a285b7834a120869557fe198d51d7cfe59754c8bda9c/cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3", size = 4745252, upload-time = "2026-07-31T14:24:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/85/4f/0fa8c2f4428198f15d9ff8d63400e27afbf94ce833f6108da1eb3753f945/cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f", size = 4728939, upload-time = "2026-07-31T14:24:27.994Z" }, + { url = "https://files.pythonhosted.org/packages/d1/63/54dd723490ba2dc09b299682c10b38db38f159728bcaae8c591b8af2f22d/cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5", size = 4748483, upload-time = "2026-07-31T14:24:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/1d/dd/7c77d26285cc7f6991efce64a0f5b4f9383bfa5dd8c5033003eaf7db4cdb/cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f", size = 5367599, upload-time = "2026-07-31T14:24:32.457Z" }, + { url = "https://files.pythonhosted.org/packages/46/c9/f60aed34c013f317f92817b6c171c2d22a78270fa41109bd4b08af26b194/cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025", size = 4762647, upload-time = "2026-07-31T14:24:34.599Z" }, + { url = "https://files.pythonhosted.org/packages/be/f3/f9a0173b139372c3a48ed98154b45cc6b9de17c789d5ab552e621c293609/cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a", size = 4385197, upload-time = "2026-07-31T14:24:36.647Z" }, + { url = "https://files.pythonhosted.org/packages/d8/36/83bb81f6e569bc38e1e4a7bc80f29b46bb9601920bc455fc8e888f5d5742/cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b", size = 4748095, upload-time = "2026-07-31T14:24:39.493Z" }, + { url = "https://files.pythonhosted.org/packages/6b/16/d3008eff98c764979865834c3d386d4fd041b5f52e7f34fc29ac1a5eb515/cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708", size = 5325948, upload-time = "2026-07-31T14:24:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f8/d97f9603efda3888187bfdb893f26c41be4735c10631d05d284ee6b047c4/cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47", size = 4762400, upload-time = "2026-07-31T14:24:43.636Z" }, + { url = "https://files.pythonhosted.org/packages/64/a2/4615c8f7d81a00b1d6e6afe19f694e1543582349fb5f4076f6cb5dc36485/cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9", size = 4878208, upload-time = "2026-07-31T14:24:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/d2/1a/efcfb02f91407149a0dacffffab791f7e19bf6385f63b3666dc8b5e5c9c8/cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7", size = 5037050, upload-time = "2026-07-31T14:24:47.697Z" }, + { url = "https://files.pythonhosted.org/packages/57/30/4a22984d4f1bdfb8c054f07a92bc176b97a3134cc1d6c4b3bffb1f3688b4/cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba", size = 3874135, upload-time = "2026-07-31T14:24:50.085Z" }, + { url = "https://files.pythonhosted.org/packages/9d/3e/e54cde8c01631a5a8226ccd617eab9e57fd5cfdad90f1a9e6bb570794631/cryptography-50.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5e34edd123674534acd70147f0ca331eaa2c74e6325fb2028c886aa26ba0b68c", size = 3963170, upload-time = "2026-07-31T14:24:51.968Z" }, + { url = "https://files.pythonhosted.org/packages/01/b6/0b9e125e90f3d2dcf599a218a899cda7326a3158cfa258723f0b398b08f6/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8eb5e1172eb569ea8a872796576e6a67c276351728b6455d5beb01242b027c6a", size = 4692441, upload-time = "2026-07-31T14:24:53.743Z" }, + { url = "https://files.pythonhosted.org/packages/53/c9/a5151588710785a96d7bc4de27d4cd62f263bbbcb203cfe29df537eb6505/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:910d11e1a385c654bf738bf3e6b8e6ed5de0f5610fcae2be9e5b398d8081d20e", size = 4699810, upload-time = "2026-07-31T14:24:55.746Z" }, + { url = "https://files.pythonhosted.org/packages/c7/1a/15b92b25eb6ce3089cd49377ae990a0f3ad485a510f968aed1f19dbdcdf2/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:62598a8a57f815db4c6259a4e97d857dab56697e7de8e8ab02352ab74da1995d", size = 4691924, upload-time = "2026-07-31T14:24:58.082Z" }, + { url = "https://files.pythonhosted.org/packages/62/15/219075012ab13e8905f3cd572204f4acb4b111df787104346b9bc0cea789/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:07479a1cb08219ab719147e742e76090c9c773321959bb94946fffdd397a6437", size = 4699593, upload-time = "2026-07-31T14:24:59.951Z" }, + { url = "https://files.pythonhosted.org/packages/8e/b5/c2c5fce26f0ee40d21bafe7f191d29a34b35a65ac4fe8a1191d1983612e9/cryptography-50.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c99c003e088647b8a5b7c145d6f78c335f6348332b62e142d411c4b63d1460b9", size = 3813796, upload-time = "2026-07-31T14:25:02.298Z" }, +] + [[package]] name = "distlib" version = "0.4.0" diff --git a/docker-compose.yml b/docker-compose.yml index 25da1206..2a5fb32f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,6 +82,7 @@ services: - AUTH_PASSWORD=admin12345 - JWT_SECRET=your_jwt_secret_key_change_in_production_please - ACCESS_TOKEN_EXPIRE_HOURS=24 + - CONNECTOR_SECRET_KEY=${CONNECTOR_SECRET_KEY:-} # Serving / pool sizing. UVICORN_WORKERS is read both by the command # below and by app.core.config, so the process count and the connection # budget can never drift apart. @@ -127,6 +128,8 @@ services: - S3_ACCESS_KEY=fake - S3_SECRET_KEY=fakefake - S3_REGION=us-east-1 + - CONNECTOR_SECRET_KEY=${CONNECTOR_SECRET_KEY:-} + - ANNOTATION_API_INTERNAL_URL=${ANNOTATION_API_INTERNAL_URL:-http://annotation_api:5050} restart: unless-stopped command: "procrastinate --app=app.worker.app worker" healthcheck: diff --git a/docs/specs/2026-08-06-alert-api-connector-design.md b/docs/specs/2026-08-06-alert-api-connector-design.md new file mode 100644 index 00000000..b924ddf6 --- /dev/null +++ b/docs/specs/2026-08-06-alert-api-connector-design.md @@ -0,0 +1,393 @@ +# Alert API Connector — Design + +**Date:** 2026-08-06 +**Status:** Approved for planning + +## Problem + +Importing from an alert API is a manual, per-operator ritual: + +```bash +make import-alert-api DATE_FROM=2026-08-05 DATE_END=2026-08-05 +``` + +The script runs on someone's laptop, reads four credentials from +`annotation_api/.env`, and imports whatever the `ALERT_API_LOGIN` account happens +to see. Consequences: + +- **Nobody imports daily.** It only happens when a person remembers. +- **Organization scope is implicit.** The account you log in as decides what you + get. There is no way to say "ingest Ardèche and Gard, not Aveyron". +- **Coverage is invisible.** Nothing records which days were imported for which + organization, so gaps are found by accident. +- **Adding an alert API means editing `.env` and redeploying.** + +## Goal + +A *connector*: a stored, credentialed link to one alert API. You plug it in +through the UI, pick which of its organizations to ingest, and the annotation +API imports them daily on its own. The page then shows you, per organization +and per day, what was covered. + +The connector page is **configuration and observation, not a control panel** — +there is no "run now" button. Backfill remains a CLI job. + +## Architecture + +``` +┌──────────────┐ verify / discover orgs ┌───────────────┐ +│ Frontend │ ──────────────────────────▶ │ Annotation │ +│ /connectors │ ◀────────── coverage ────── │ API │ +└──────────────┘ └───────┬───────┘ + │ DB + ┌───────▼───────┐ + alert API ◀───── fetch ──────────── │ Worker │ + │ (procrastinate)│ + annotation API ◀──── POST ────────── └───────────────┘ +``` + +The worker holds the schedule and the import loop. It reuses the existing +importer as a library and posts through the annotation API's own HTTP +endpoints, which already handle image transfer and 409 reconciliation. + +## Data model + +Three new tables. + +### `alert_api_connector` + +One row per alert API. + +| field | type | notes | +| --- | --- | --- | +| `id` | int PK | | +| `name` | str | e.g. `Pyronear France` | +| `base_url` | str, unique | e.g. `https://alertapi.pyronear.org` | +| `source_api` | `SourceApi` enum, **unique** | see below | +| `login` | str | alert API account (expected to be an admin) | +| `password_encrypted` | str | Fernet token; never returned by the API | +| `is_enabled` | bool | pause without deleting | +| `trailing_days` | int, default 3 | size of the re-imported window | +| `image_transfer` | str \| null | `url` / `bucket-copy` / null = importer's auto-detect | +| `last_verified_at` | datetime \| null | set by the verify action | +| `last_verify_error` | str \| null | | +| `created_at`, `updated_at` | datetime | | + +**`source_api` is unique across connectors.** Sequence identity is +`(alert_api_id, source_api)` and alert identity is +`(source_api, platform_alert_id)`; two connectors sharing a `source_api` would +let alert ids from different platforms collide. + +Adding a *new* platform stays a code change (enum value + Alembic migration). +`source_api` is referenced throughout the frontend's filters, so making it +free-form has a far larger blast radius than this feature justifies. + +### `alert_api_connector_organization` + +Organizations discovered on a connector. + +| field | type | notes | +| --- | --- | --- | +| `id` | int PK | | +| `connector_id` | FK → connector, cascade delete | | +| `organization_id` | int | the id on the **remote** alert API | +| `name` | str | cached at discovery | +| `is_enabled` | bool, default false | ingest this org or not | +| `enabled_at` | datetime \| null | first time it was enabled | + +Unique on `(connector_id, organization_id)`. + +### `alert_api_import_coverage` + +**One row per heatmap cell.** + +| field | type | notes | +| --- | --- | --- | +| `id` | int PK | | +| `connector_id` | FK → connector, cascade delete | | +| `organization_id` | int | remote org id | +| `covered_date` | date | UTC date on the alert API | +| `status` | `ok` / `partial` / `failed` | see below | +| `alerts_fetched` | int | alert-API sequences seen for this org that day | +| `alerts_imported` | int | newly imported | +| `alerts_skipped` | int | already present (short-circuited) | +| `alerts_failed` | int | errored during import | +| `lanes_created` | int | annotation sequences created (object-split fan-out) | +| `error` | str \| null | | +| `last_attempt_at` | datetime | | + +Unique on `(connector_id, organization_id, covered_date)`; upserted on every +attempt. + +`status` is derived, not free-form: + +- `failed` — nothing was imported for this org that day: the connector itself + errored, or every alert failed. +- `partial` — `alerts_failed > 0` **and** at least one alert imported or skipped. +- `ok` — everything else, including a day with zero alerts. + +Two behaviours follow from this shape: + +- An org with **zero alerts** that day still gets a row with counts `0` and + status `ok`. That is what distinguishes "we looked, nothing was there" (grey) + from "we never got there" (hatched). +- A **connector-level failure** (bad credentials, alert API down) writes + `failed` rows for *every* enabled org on that date. This is why no separate + run-history table is needed — failures always have a cell to land in. + +Counts are in **alerts** (alert-API sequences), not annotation lanes, because +that is the unit a human reasons about. `lanes_created` records the fan-out +separately. + +## Credentials + +New `app/services/secrets.py` (~15 lines): `encrypt_secret` / `decrypt_secret` +over `Fernet(settings.CONNECTOR_SECRET_KEY)`. Adds `cryptography` as a +dependency. + +- The read schema exposes `has_password: bool`, never the value. +- The write schema accepts `password` only on create or an explicit replace. +- If `CONNECTOR_SECRET_KEY` is unset, connector create/update returns **400** + with a message naming the variable, and the worker logs and skips affected + connectors. Existing deployments keep working untouched until someone opts in. + +The threat this addresses is a **database dump** — a backup, a copy pulled for +debugging — not an attacker with a shell on the host (who would read the env +too). Losing the key means re-entering credentials through the UI. + +**One credential pair, not two.** Today's importer holds a regular login *and* +an admin login. The premise here is that the admin account alone covers both +roles. The verify action tests that premise empirically rather than assuming it +(see *Verify*). If sequence listing turns out to be organization-scoped, the org +table is where per-org credentials would later hang — no reshaping needed. + +## Worker: authentication to its own API + +The importer POSTs to the annotation API over HTTP. The worker therefore needs +a token — but **not a password**. + +`create_access_token` is a pure function over `settings.JWT_SECRET` +(`app/auth/dependencies.py:34`), which the worker already has, and the worker +already resolves the worker user by name (`app/worker.py:159`): + +```python +worker_user = await UserCRUD(session).get_by_username(settings.WORKER_USERNAME) +token = create_access_token({"sub": worker_user.username, "user_id": worker_user.id}) +``` + +This avoids a plaintext password in the worker's environment (the exact thing +Fernet encryption exists to prevent), avoids duplicating a credential to talk to +itself, and removes the cold-boot race where the worker starts before the API +seeds its users. + +**Known coupling:** the worker user is seeded **inactive** on purpose so login +rejects it (`app/main.py:71`). A minted token works because the sequence and +detection endpoints depend on `get_current_user`, which does not check +`is_active` — only `get_current_active_user` does. A pinning test locks this in +so a future auth tightening fails CI instead of silently breaking nightly +imports. + +The only new setting is `ANNOTATION_API_INTERNAL_URL` (`http://api:5050` in +compose), which is not a secret. + +## Importer refactor + +`scripts/data_transfer/ingestion/alert_api/import.py` is refactored, with **no +behaviour change to the CLI**: + +- Extract an `ImportConfig` dataclass and `run_import(config) -> ImportResult` + holding everything `main()` currently does after argument parsing. +- `main()` becomes: parse argv + read env → build `ImportConfig` → `run_import` + → render the console summary. + +`ImportConfig` gains two fields the CLI does not currently expose: + +- `organization_ids: set[int] | None` — restrict to enabled organizations +- `skip_platform_alert_ids: set[int]` — alerts already in the database + +`ImportConfig` keeps the importer's existing *two* credential slots (regular and +admin). A connector stores one pair and passes it to both, which is precisely +the premise verify tests; the CLI continues to fill them from the four existing +environment variables. + +**Both filters apply immediately after the day's sequence listing, before any +detection fetch.** The camera index (built once from `list_cameras`) resolves +`camera_id → organization_id`, so filtering is a dict lookup. + +This is the short-circuit: re-running an already-imported day costs **one +listing call and zero detection calls**. Today the importer discovers "already +exists" only at POST time (`shared.py:554`), after paying for every detection +fetch. + +`ImportResult` gains **per-organization counters**; today's statistics are +global only, and the coverage rows need the breakdown. + +`risk_score="extreme"` is retained deliberately — it neutralizes the alert API's +FWI filter so low-risk sequences are not dropped. + +**Dockerfile:** add `COPY scripts /app/scripts` to the builder stage. +`PYTHONPATH=/app` is already set, and `requests`, `rich`, and `tqdm` are already +main dependencies, so nothing else changes. + +## Worker: scheduling and import + +Two tasks in `app/worker.py`, following the existing periodic-sweep pattern. + +### `schedule_connector_imports` — `@app.periodic(cron="0 3 * * *")` + +Fires once a day. Defers one `run_connector_import` job for every enabled +connector that has at least one enabled organization, each with +`queueing_lock=f"connector-import-{id}"` so a still-running connector cannot +have a second job queued behind it. + +No "already ran today" bookkeeping is needed: procrastinate defers a periodic +task once per cron period, and the `queueing_lock` covers the overlap case. + +**Why not an hourly sweep with a per-connector run hour?** That only buys the +ability to stagger connectors across the night, and it is not what makes the +schedule robust — `trailing_days` is. A worker that is down at 03:00 loses +nothing, because the next day's run re-covers that date inside its window. If +staggering is ever needed (several heavy connectors competing for the same +hour), it is a small change: add `run_at_hour_utc`, move the cron to `0 * * * *`, +and match on the current hour. + +The run hour is fixed at deploy time, since `@app.periodic` takes a static cron +expression. + +### `run_connector_import(connector_id)` + +1. Load the connector and its enabled organizations; decrypt the password (log + and bail if `CONNECTOR_SECRET_KEY` is missing). +2. Build the skip set in one query: + `SELECT DISTINCT platform_alert_id FROM sequence WHERE source_api = :src`. + `platform_alert_id` is the alert API's own sequence id, shared by every lane + of an alert, and already indexed as `ix_sequence_platform_alert_id` + (`app/models.py:197`). +3. For each date in `[today − trailing_days, today − 1]` (UTC), call `run_import` + via `asyncio.to_thread` — the importer is synchronous `requests`, and this + keeps it off the event loop. +4. Upsert coverage rows per enabled organization from the per-org statistics. On + a connector-level failure, write `failed` rows for every enabled organization + on that date. + +A manual `make import-alert-api` running concurrently with a scheduled job is +wasteful but not corrupting — both skip alerts that already exist. + +## API + +All endpoints under `/api/v1/connectors/`, gated by the existing +`get_current_superuser`. + +| endpoint | purpose | +| --- | --- | +| `GET /` | list connectors with `has_password`, last import, org counts | +| `POST /` | create | +| `PATCH /{id}` | update; `password` optional | +| `DELETE /{id}` | delete (cascades to orgs and coverage) | +| `POST /{id}/verify` | log in, discover orgs, report reachability | +| `PATCH /{id}/organizations/{org_id}` | toggle `is_enabled` | +| `GET /{id}/coverage?date_from=&date_end=` | heatmap data | + +### Verify + +Runs in the API process via `asyncio.to_thread` with a timeout, since the user +is waiting on it. It: + +1. Authenticates; on failure records `last_verify_error` and returns it. +2. Calls `list_organizations` and **upserts** org rows (idempotent; re-verifying + never duplicates or resets `is_enabled`). +3. Lists yesterday's sequences with the connector's token, maps + `camera_id → organization_id` via `list_cameras`, and reports **"saw + sequences from N of M organizations"**. + +Point 3 is how the cross-organization premise gets tested against reality +instead of assumed. It is reported as a count rather than a boolean on purpose: +one organization on a quiet day proves nothing, but 4-of-7 proves cross-org +listing works. + +## Frontend + +Route `/connectors`, superuser-gated and nav-linked exactly like +`UserManagementPage` (`frontend/src/pages/UserManagementPage.tsx:81`, +`frontend/src/components/layout/AppLayout.tsx:224`). Server state via TanStack +Query; no Zustand needed. + +**List page:** name, base URL, `source_api` badge, enabled toggle, last +verified, most recent covered date, and "3 of 7 organizations enabled". + +The schedule form carries only `trailing_days`; the daily run hour is deployment +configuration, not per-connector state. + +**Detail page**, top to bottom: + +1. Credentials and schedule form. The password field renders as *set* with a + Replace action and is never populated from the server. +2. "Verify & discover organizations" — result banner plus the organization + checkbox list. +3. **Coverage heatmap**: organizations as rows, days as columns, over a + selectable window (default 30 days). + +Heatmap cell states: + +| state | appearance | +| --- | --- | +| imported, count > 0 | green, intensity by volume | +| covered, 0 alerts | grey | +| `partial` | green with a warning marker | +| `failed`, or no row at all | hatched red | +| before the org's `enabled_at` | dashed outline | + +`failed` and "no row" share an appearance because both mean *we do not have this +day*; the tooltip distinguishes them, showing the recorded error for `failed` +and "never attempted" otherwise. Hover otherwise shows date, organization, and +counts. + +## Testing + +**Backend** + +- Fernet round-trip; missing `CONNECTOR_SECRET_KEY` → 400 on create. +- CRUD: non-superuser gets 403; the password never appears in any response. +- Verify against a mocked alert API: discovers organizations, upserts + idempotently without resetting `is_enabled`, records the error on bad + credentials. +- Coverage: upsert idempotency; a zero-alert day writes `ok`; a connector + failure writes `failed` for every enabled organization. +- Sweep: defers a job per enabled connector, and skips both disabled connectors + and connectors with no enabled organizations. +- **Pinning test:** a worker-minted token can POST a sequence (guards the + inactive-worker-user coupling). +- **Short-circuit test:** `skip_platform_alert_ids` causes **zero** detection + fetches for already-imported alerts. +- Importer: the CLI's argv/env → `ImportConfig` mapping is unchanged. + +**Frontend** (Vitest, under `frontend/tests/`) + +- Heatmap renders every cell state, including not-enabled and failed. +- Organization toggle mutation. +- Password field write-only behaviour. +- Superuser gating. + +## Risks and open questions + +1. **Cross-organization admin listing is unverified.** The design works either + way; if listing is organization-scoped, only some organizations populate, and + the fix is per-org credentials on the organization table. Verify reports what + it actually saw. +2. **`cryptography` becomes a new dependency**, and losing + `CONNECTOR_SECRET_KEY` means re-entering credentials through the UI. +3. **Enabling an organization does not backfill.** It starts at the next run's + trailing window; earlier days stay dashed. +4. **Late-arriving frames on an already-imported alert are still never picked + up.** The skip is whole-alert, matching today's behaviour. Now at least + visible on the heatmap as a suspiciously low count. + +## Out of scope + +- A "run now" button. +- Backfill from the UI — `make import-alert-api DATE_FROM=… DATE_END=…` covers + it, and a date-range picker in a browser is an easy way to launch an enormous + job by accident. +- Per-organization credentials. +- New `SourceApi` enum values. +- Re-ingesting alerts that already exist. diff --git a/docs/specs/2026-08-07-connector-test-connection-design.md b/docs/specs/2026-08-07-connector-test-connection-design.md new file mode 100644 index 00000000..5ad4a915 --- /dev/null +++ b/docs/specs/2026-08-07-connector-test-connection-design.md @@ -0,0 +1,70 @@ +# Connector "Test connection" button + +2026-08-07 · extends the alert API connector feature (PR #337, `docs/specs/2026-08-06-alert-api-connector-design.md`) + +## Problem + +Credentials entered in the create-connector form are only testable *after* saving, via +the detail page's verify — which requires a stored connector id and the encrypted +password. Production probing (2026-08-07) showed the failure mode operators will +actually hit: an org-scoped alert-API credential authenticates fine, then fails at +`/organizations/` with `{"detail": "Incompatible token scope."}`. Nothing in the UI +states that the credential must be admin-scoped; a pre-save test is where that surfaces +naturally. + +## Backend + +`POST /api/v1/connectors/test` — superuser-only, stateless, no DB access. + +- Request: `{base_url, login, password}` (plaintext password in the request body only; + never logged, never persisted). +- Behavior: token exchange, then `list_organizations`, in a thread via + `asyncio.to_thread` like verify's probe. Reuses the same non-list shape guard as + `connector_verify.py`, so an org-scoped credential reports + `alert API returned an unexpected organizations response: Incompatible token scope.` +- Response: `{ok: true, organizations_total: N}` or `{ok: false, error: "..."}`. + Never raises for an unreachable/unauthorized alert API — a human is watching. +- Timeout: whole probe wrapped in `asyncio.wait_for` at **25 s**, deliberately under + the frontend's global 30 s axios timeout so the backend always answers before the + browser gives up (verify's ~100 s bound predates this and remains a known follow-up). +- Implementation lives in `connector_verify.py` next to `verify_connector`, sharing the + shape-guard logic. + +## Frontend + +A "Test connection" button inside the create-connector form (`ConnectorsPage.tsx`): + +- Enabled once base URL, login, and password are all non-empty. +- Click → pending state → inline result under the credentials fields: + green "Connection OK — N organizations visible" or the backend's error text verbatim + in red. +- Advisory only: does not gate Create. An operator can save untested (e.g. while the + alert API is down). +- The result clears whenever base URL, login, or password change, so a stale green + cannot vouch for edited credentials. + +### Rider: full-width connectors table + +`ConnectorsPage` caps its root at `mx-auto max-w-5xl`, which squeezes the 7-column +table into its `overflow-x-auto` fallback (horizontal scroll). The groups list page — +the reference table layout — uses a full-width `space-y-6` root. Align the connectors +page with it: drop the width cap and let the app layout's padding govern, keeping the +`overflow-x-auto` wrapper as the narrow-viewport fallback. The create-form modal keeps +its own `max-w-md`. + +## Out of scope + +- No test button on the edit path — there, an omitted password means "keep the stored + credential", so there is nothing to test client-side. +- No auto-test on create, no result caching, no UI copy changes beyond the button and + its result line. + +## Testing + +- Backend endpoint tests, client stubbed at the same seam as the existing verify tests: + superuser gate, ok path (count reported), bad-password path, scope-error path with the + detail passed through, timeout path. +- Frontend form tests: button disabled until all three fields filled, pending state, + success and error rendering, result cleared on credential edit. +- Layout rider: assert the page root no longer carries the `max-w-5xl` cap (mirrors the + groups list page's root). diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c5f69775..594501ca 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -12,6 +12,8 @@ import LocalizeAlertPage from '@/pages/LocalizeAlertPage'; import SequenceGroupAnnotatePage from '@/pages/SequenceGroupAnnotatePage'; import SequenceGroupsListPage from '@/pages/SequenceGroupsListPage'; import UserManagementPage from '@/pages/UserManagementPage'; +import ConnectorsPage from '@/pages/ConnectorsPage'; +import ConnectorDetailPage from '@/pages/ConnectorDetailPage'; import GuidePage from '@/pages/GuidePage'; import LoginPage from '@/pages/LoginPage'; import { legacyRedirectRoutes } from '@/components/routing/legacyRedirects'; @@ -151,6 +153,8 @@ function App() { {legacyRedirectRoutes} } /> + } /> + } /> } /> diff --git a/frontend/src/components/connectors/CoverageHeatmap.tsx b/frontend/src/components/connectors/CoverageHeatmap.tsx new file mode 100644 index 00000000..3044245a --- /dev/null +++ b/frontend/src/components/connectors/CoverageHeatmap.tsx @@ -0,0 +1,147 @@ +import { useState } from 'react'; + +import type { ConnectorOrganization, CoverageCell } from '@/types/api'; + +type CellState = 'imported' | 'empty' | 'partial' | 'failed' | 'missing' | 'not-enabled'; + +interface Props { + organizations: ConnectorOrganization[]; + cells: CoverageCell[]; + dateFrom: string; + dateEnd: string; +} + +/** + * Inclusive list of ISO dates, built in UTC so a viewer's timezone can never + * shift which day a cell represents — coverage dates are alert-API UTC dates. + */ +function isoDateRange(from: string, to: string): string[] { + const dates: string[] = []; + const cursor = new Date(`${from}T00:00:00Z`); + const end = new Date(`${to}T00:00:00Z`); + while (cursor <= end) { + dates.push(cursor.toISOString().slice(0, 10)); + cursor.setUTCDate(cursor.getUTCDate() + 1); + } + return dates; +} + +function cellState( + cell: CoverageCell | undefined, + enabledAt: string | null, + day: string +): CellState { + if (!cell) { + if (enabledAt && day < enabledAt.slice(0, 10)) return 'not-enabled'; + return 'missing'; + } + if (cell.status === 'failed') return 'failed'; + if (cell.status === 'partial') return 'partial'; + // Key on total coverage (alerts_fetched), not fresh imports + // (alerts_imported): a re-run inside the trailing window re-fetches the + // same alerts and files them as skipped rather than imported (see + // runner.py), so alerts_imported alone would flip a fully-covered day back + // to "empty" the moment it ages out of the newest run. + return cell.alerts_fetched > 0 ? 'imported' : 'empty'; +} + +// failed and missing share an appearance: both mean "we do not have this day". +// The tooltip is what distinguishes them. Signal is legitimate here — DESIGN.md +// restricts it to "errors, destructive, attention only", and a hole in coverage +// is exactly that. Same hatch treatment and rgb value as the "no source found" +// hole in ObjectFilmstrip.tsx (signal token #B3261E → rgb(179,38,30)), not +// Tailwind's default red. +const HATCHED = + 'bg-signal-soft [background-image:repeating-linear-gradient(45deg,transparent,transparent_3px,rgba(179,38,30,0.55)_3px,rgba(179,38,30,0.55)_6px)]'; + +const STATE_CLASS: Record = { + // Positive state → pine (DESIGN.md palette row: "Localize lane identity, + // active nav, positive states"). + imported: 'bg-pine', + // Some imported, some failed: pine fill with a signal ring marker rather + // than a second full-fill accent, keeping to "one accent per element" + // while still surfacing the attention-worthy part. + partial: 'bg-pine ring-2 ring-inset ring-signal', + // Covered with zero alerts is a success state — the import ran, the day + // is covered, there was just nothing to fetch. Pale pine keeps it in the + // green family (a quiet cousin of imported) instead of reading as an + // absence: the previous ash square was indistinguishable from the dashed + // not-enabled outline at 16px, which is now the only grey state. + empty: 'bg-pine-soft ring-1 ring-inset ring-pine', + failed: HATCHED, + missing: HATCHED, + 'not-enabled': 'border border-dashed border-line', +}; + +function tooltip(state: CellState, day: string, orgName: string, cell?: CoverageCell): string { + const head = `${orgName} — ${day}`; + if (state === 'not-enabled') return `${head}: organization not enabled yet`; + if (state === 'missing') return `${head}: never attempted`; + if (state === 'failed') return `${head}: failed — ${cell?.error ?? 'unknown error'}`; + const counts = `${cell?.alerts_imported ?? 0} imported, ${cell?.alerts_skipped ?? 0} skipped`; + if (state === 'partial') return `${head}: ${counts}, ${cell?.alerts_failed ?? 0} failed`; + return `${head}: ${counts}`; +} + +export function CoverageHeatmap({ organizations, cells, dateFrom, dateEnd }: Props) { + const days = isoDateRange(dateFrom, dateEnd); + const byKey = new Map(cells.map(c => [`${c.organization_id}:${c.covered_date}`, c])); + // One shared hover tooltip, position: fixed. A per-cell CSS bubble would be + // clipped by the overflow-x-auto scroll container (which clips vertically + // too), and 16px cells leave no room inside it; fixed positioning escapes + // the container without a portal. Viewport coordinates come from the cell's + // own rect at mouseenter. + const [hover, setHover] = useState<{ text: string; x: number; y: number } | null>(null); + + return ( +
+ + + {organizations.map(org => ( + + {/* Organization name, not a numeral — font-body per the "primary + cell" table recipe, not font-data (that's for counts/dates/ + headers in a thead, which this table has none of). */} + + {days.map(day => { + const cell = byKey.get(`${org.organization_id}:${day}`); + const state = cellState(cell, org.enabled_at, day); + const label = tooltip(state, day, org.name, cell); + return ( + + ); + })} + + ))} + +
+ {org.name} + +
{ + const rect = e.currentTarget.getBoundingClientRect(); + setHover({ text: label, x: rect.left + rect.width / 2, y: rect.top }); + }} + onMouseLeave={() => setHover(null)} + className={`h-4 w-4 rounded-sm ${STATE_CLASS[state]}`} + /> +
+ {hover && ( +
+ {hover.text} +
+ )} +
+ ); +} diff --git a/frontend/src/components/layout/AppLayout.tsx b/frontend/src/components/layout/AppLayout.tsx index c4b71b04..bc12ed98 100644 --- a/frontend/src/components/layout/AppLayout.tsx +++ b/frontend/src/components/layout/AppLayout.tsx @@ -1,7 +1,7 @@ import { useState } from 'react'; import { Link, useLocation } from 'react-router-dom'; import { Menu } from '@headlessui/react'; -import { Menu as MenuIcon, MoreVertical, X, LogOut, User, Users } from 'lucide-react'; +import { Menu as MenuIcon, MoreVertical, X, LogOut, User, Users, Plug } from 'lucide-react'; import { clsx } from 'clsx'; import { useAnnotationCounts } from '@/hooks/useAnnotationCounts'; import NotificationBadge from '@/components/ui/NotificationBadge'; @@ -216,15 +216,15 @@ function UserSection() { - + {isSuperuser() && ( {({ active }) => ( @@ -233,13 +233,29 @@ function UserSection() { )} )} + {isSuperuser() && ( + + {({ active }) => ( + + + Connectors + + )} + + )} {({ active }) => ( + + ) : ( +
+ setPasswordValue(e.target.value)} + placeholder="New password" + className={inputClass} + /> + +
+ )} + + +
+ + + setFormData(prev => + prev ? { ...prev, trailing_days: Number(e.target.value) } : prev + ) + } + className={inputClass} + /> +
+ +
+ + +
+ + + +
+ +
+ + )} + + + {/* Verify */} +
+

+ Verify & discover organizations +

+

+ Lists sequences across every organization visible to this account, and records which ones + it actually saw — one admin login is expected to cover them all. +

+
+ + + Last verified:{' '} + {connector.last_verified_at ? formatDateTime(connector.last_verified_at) : 'Never'} + +
+ + {verifyResult && ( +
+ {verifyResult.ok ? ( +

+ Saw sequences from{' '} + + {verifyResult.organizations_seen_in_sample} of {verifyResult.organizations_total} + {' '} + organizations on{' '} + + {/* sample_date is a bare UTC date string ('2026-08-05'), not a + timestamp — formatDate would re-parse it as UTC midnight and + render it a day early for negative-offset viewers. Render it + raw, same treatment as dateFrom/dateEnd below. */} + {verifyResult.sample_date ?? '—'} + + . +

+ ) : ( +
+ {verifyResult.error || 'Verification failed'} +
+ )} +
+ )} + + {verifyMutation.isError && ( +
+ {(verifyMutation.error as { detail?: string })?.detail || 'Verification request failed'} +
+ )} +
+ + {/* Organizations */} +
+

Organizations

+

+ Enable the organizations this connector should import from. +

+ {!organizations || organizations.length === 0 ? ( +

+ No organizations discovered yet. Run verify to discover them. +

+ ) : ( +
    + {organizations.map(org => ( +
  • +
    + + toggleMutation.mutate({ + organizationId: org.organization_id, + isEnabled: !org.is_enabled, + }) + } + className="rounded border-line text-ember focus:ring-ember" + /> + +
    + + {org.is_enabled && org.enabled_at + ? `enabled ${formatDate(org.enabled_at)}` + : 'not enabled'} + +
    • + ))} +
+ )} +
+ + {/* Coverage */} +
+

Coverage

+

+ {dateFrom} to {dateEnd} +

+ + {/* Legend — the six cell states are unreadable without it. Swatches + mirror CoverageHeatmap's STATE_CLASS exactly (same pine / + pine-soft / signal-hatch / dashed treatment), duplicated here + rather than imported since the component doesn't export its class + constants. */} +
+ + + Imported + + + + Partially failed + + + + Covered, no alerts + + + + Failed or never attempted + + + + Not enabled yet + +
+ + {enabledOrganizations.length === 0 ? ( +

No organizations enabled yet.

+ ) : ( +
+ +
+ )} +
+ + ); +} diff --git a/frontend/src/pages/ConnectorsPage.tsx b/frontend/src/pages/ConnectorsPage.tsx new file mode 100644 index 00000000..681e0d6f --- /dev/null +++ b/frontend/src/pages/ConnectorsPage.tsx @@ -0,0 +1,431 @@ +import { useState } from 'react'; +import { Link } from 'react-router-dom'; +import { Plus, Plug, Trash2, X } from 'lucide-react'; +import { clsx } from 'clsx'; + +import { + useConnectors, + useCreateConnector, + useDeleteConnector, + useTestConnector, +} from '@/hooks/useConnectors'; +import { useAuthStore } from '@/store/useAuthStore'; +import { formatDateTime } from '@/utils/datetime'; +import type { Connector, ConnectorCreatePayload } from '@/types/api'; + +const SOURCE_API_OPTIONS: { value: string; label: string }[] = [ + { value: 'pyronear_french', label: 'Pyronear (French)' }, + { value: 'alert_wildfire', label: 'AlertWildfire' }, + { value: 'api_cenia', label: 'CENIA' }, +]; + +const IMAGE_TRANSFER_OPTIONS: { value: '' | 'url' | 'bucket-copy'; label: string }[] = [ + { value: '', label: 'Auto' }, + { value: 'url', label: 'URL' }, + { value: 'bucket-copy', label: 'Bucket copy' }, +]; + +export default function ConnectorsPage() { + const { isSuperuser } = useAuthStore(); + + // ALL HOOKS MUST BE BEFORE ANY CONDITIONAL RETURNS + const { data: connectors, isLoading } = useConnectors(); + const createConnectorMutation = useCreateConnector(); + const deleteConnectorMutation = useDeleteConnector(); + const [showCreateModal, setShowCreateModal] = useState(false); + + // Check if current user is superuser - AFTER all hooks + if (!isSuperuser()) { + return ( +
+
+ +

Access denied

+

+ You need superuser privileges to access this page. +

+
+
+ ); + } + + const handleDelete = (connector: Connector) => { + if (confirm(`Are you sure you want to delete connector "${connector.name}"?`)) { + deleteConnectorMutation.mutate(connector.id); + } + }; + + if (isLoading) { + return ( +
+
+
+ ); + } + + return ( +
+ {/* Header */} +
+
+

+ Connectors +

+

+ Alert APIs the backend imports from daily +

+
+ +
+ + {/* Connectors */} + {!connectors || connectors.length === 0 ? ( +
+ +

+ No connectors configured yet. +

+
+ ) : ( +
+
+ + + + + + + + + + + + + + {connectors.map(connector => ( + + + + + + + + + + ))} + +
+ Name + + Base URL + + Source + + Status + + Organizations + + Verification + + Actions +
+ + {connector.name} + + + {connector.base_url} + + + {connector.source_api} + + + + {connector.is_enabled ? 'Enabled' : 'Disabled'} + + + {connector.organizations_enabled} of {connector.organizations_total}{' '} + organizations + + {connector.last_verify_error ? ( + + {connector.last_verify_error} + + ) : ( + + {connector.last_verified_at + ? formatDateTime(connector.last_verified_at) + : 'Never'} + + )} + + +
+
+
+ )} + + {showCreateModal && ( + setShowCreateModal(false)} + onSubmit={data => { + createConnectorMutation.mutate(data, { + onSuccess: () => setShowCreateModal(false), + }); + }} + isLoading={createConnectorMutation.isPending} + error={createConnectorMutation.error as { detail?: string } | null} + /> + )} +
+ ); +} + +function CreateConnectorModal({ + onClose, + onSubmit, + isLoading, + error, +}: { + onClose: () => void; + onSubmit: (data: ConnectorCreatePayload) => void; + isLoading: boolean; + error: { detail?: string } | null; +}) { + const [formData, setFormData] = useState({ + name: '', + base_url: '', + source_api: SOURCE_API_OPTIONS[0].value, + login: '', + password: '', + is_enabled: true, + trailing_days: 3, + image_transfer: null, + }); + + const testMutation = useTestConnector(); + const canTest = + formData.base_url.trim() !== '' && + formData.login.trim() !== '' && + formData.password !== '' && + !testMutation.isPending; + + const setCredentialField = (field: 'base_url' | 'login' | 'password', value: string) => { + testMutation.reset(); // a stale result must not vouch for edited credentials + setFormData(prev => ({ ...prev, [field]: value })); + }; + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + onSubmit(formData); + }; + + const inputClass = + 'w-full rounded-lg border border-line px-3 py-2 font-body text-sm text-char focus:outline-none focus:ring-2 focus:ring-ember focus:border-ember transition-colors'; + const labelClass = 'mb-1 block font-body text-xs font-medium text-haze'; + + return ( +
+
+
+

Add connector

+ +
+ + {error && ( +
+ {error.detail || 'Failed to create connector'} +
+ )} + +
+
+ + setFormData(prev => ({ ...prev, name: e.target.value }))} + className={inputClass} + /> +
+ +
+ + setCredentialField('base_url', e.target.value)} + className={inputClass} + placeholder="https://alertapi.example.org" + /> +
+ +
+ + +
+ +
+ + setCredentialField('login', e.target.value)} + className={inputClass} + /> +
+ +
+ + setCredentialField('password', e.target.value)} + className={inputClass} + /> +
+ +
+ + {testMutation.data?.ok && ( +

+ Connection OK — {testMutation.data.organizations_total} organizations visible +

+ )} + {testMutation.data && !testMutation.data.ok && ( +

{testMutation.data.error}

+ )} + {testMutation.error && ( +

{testMutation.error.message}

+ )} +
+ +
+ + + setFormData(prev => ({ ...prev, trailing_days: Number(e.target.value) })) + } + className={inputClass} + /> +
+ +
+ + +
+ + + +
+ + +
+
+
+
+ ); +} diff --git a/frontend/src/pages/UserManagementPage.tsx b/frontend/src/pages/UserManagementPage.tsx index 2b929e5f..54326ca4 100644 --- a/frontend/src/pages/UserManagementPage.tsx +++ b/frontend/src/pages/UserManagementPage.tsx @@ -263,10 +263,14 @@ export default function UserManagementPage() { - {user.is_active ? 'Active' : 'Inactive'} + {user.is_system ? 'Service' : user.is_active ? 'Active' : 'Inactive'} diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index 3752a753..b8133aed 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -40,6 +40,14 @@ import { SequenceGroup, SequenceGroupListItem, SequenceGroupStats, + Connector, + ConnectorOrganization, + ConnectorCreatePayload, + ConnectorTestPayload, + ConnectorTestResult, + ConnectorUpdatePayload, + CoverageCell, + VerifyResult, } from '@/types/api'; import { API_ENDPOINTS } from '@/utils/constants'; @@ -627,6 +635,75 @@ class ApiClient { await this.client.delete(`/users/${id}`); } + // Alert API Connectors + async getConnectors(): Promise { + const response: AxiosResponse = await this.client.get('/connectors/'); + return response.data; + } + + async createConnector(payload: ConnectorCreatePayload): Promise { + const response: AxiosResponse = await this.client.post('/connectors/', payload); + return response.data; + } + + async updateConnector(id: number, payload: ConnectorUpdatePayload): Promise { + const response: AxiosResponse = await this.client.patch( + `/connectors/${id}`, + payload + ); + return response.data; + } + + async deleteConnector(id: number): Promise { + await this.client.delete(`/connectors/${id}`); + } + + async testConnector(payload: ConnectorTestPayload): Promise { + const response: AxiosResponse = await this.client.post( + '/connectors/test', + payload + ); + return response.data; + } + + async verifyConnector(id: number): Promise { + const response: AxiosResponse = await this.client.post( + `/connectors/${id}/verify` + ); + return response.data; + } + + async getConnectorOrganizations(id: number): Promise { + const response: AxiosResponse = await this.client.get( + `/connectors/${id}/organizations` + ); + return response.data; + } + + async toggleConnectorOrganization( + id: number, + organizationId: number, + isEnabled: boolean + ): Promise { + const response: AxiosResponse = await this.client.patch( + `/connectors/${id}/organizations/${organizationId}`, + { is_enabled: isEnabled } + ); + return response.data; + } + + async getConnectorCoverage( + id: number, + dateFrom: string, + dateEnd: string + ): Promise { + const response: AxiosResponse = await this.client.get( + `/connectors/${id}/coverage`, + { params: { date_from: dateFrom, date_end: dateEnd } } + ); + return response.data; + } + // Health check async healthCheck(): Promise<{ status: string }> { // Note: health check is at /status, not in /api/v1 diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index fa728c45..ca500b92 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -572,3 +572,80 @@ export interface ApiError { /** HTTP status of the failed response, when one was received. */ status?: number; } + +// Alert API Connector Types +export interface Connector { + id: number; + name: string; + base_url: string; + source_api: string; + login: string; + has_password: boolean; + is_enabled: boolean; + trailing_days: number; + image_transfer: 'url' | 'bucket-copy' | null; + last_verified_at: string | null; + last_verify_error: string | null; + organizations_total: number; + organizations_enabled: number; +} + +export interface ConnectorOrganization { + id: number; + organization_id: number; + name: string; + is_enabled: boolean; + enabled_at: string | null; +} + +export type CoverageStatus = 'ok' | 'partial' | 'failed'; + +export interface CoverageCell { + organization_id: number; + covered_date: string; + status: CoverageStatus; + alerts_fetched: number; + alerts_imported: number; + alerts_skipped: number; + alerts_failed: number; + lanes_created: number; + error: string | null; +} + +// organizations_seen_in_sample / organizations_total are counts, not a +// boolean: the connector design assumes one admin account can list sequences +// across every organization, and verify proves it by sampling how many +// distinct organizations actually showed up. +export interface VerifyResult { + ok: boolean; + error: string | null; + organizations: ConnectorOrganization[]; + organizations_seen_in_sample: number; + organizations_total: number; + sample_date: string | null; +} + +export interface ConnectorCreatePayload { + name: string; + base_url: string; + source_api: string; + login: string; + password: string; + is_enabled?: boolean; + trailing_days?: number; + image_transfer?: 'url' | 'bucket-copy' | null; +} + +export type ConnectorUpdatePayload = Partial>; + +export interface ConnectorTestPayload { + base_url: string; + login: string; + password: string; +} + +export interface ConnectorTestResult { + ok: boolean; + error: string | null; + organizations_total: number; +} diff --git a/frontend/src/utils/constants.ts b/frontend/src/utils/constants.ts index 75bbc53e..92148662 100644 --- a/frontend/src/utils/constants.ts +++ b/frontend/src/utils/constants.ts @@ -337,4 +337,13 @@ export const QUERY_KEYS = { USERS: ['users'], USER: (id: number) => ['users', id], ANNOTATORS: ['annotators'], + CONNECTORS: ['connectors'], + CONNECTOR_ORGANIZATIONS: (id: number) => ['connectors', id, 'organizations'], + CONNECTOR_COVERAGE: (id: number, from: string, to: string) => [ + 'connectors', + id, + 'coverage', + from, + to, + ], } as const; diff --git a/frontend/tests/components/CoverageHeatmap.test.tsx b/frontend/tests/components/CoverageHeatmap.test.tsx new file mode 100644 index 00000000..86b1eb42 --- /dev/null +++ b/frontend/tests/components/CoverageHeatmap.test.tsx @@ -0,0 +1,202 @@ +import { describe, expect, it } from 'vitest'; +import { fireEvent, render, screen } from '@testing-library/react'; + +import { CoverageHeatmap } from '@/components/connectors/CoverageHeatmap'; +import type { ConnectorOrganization, CoverageCell } from '@/types/api'; + +const ORGS: ConnectorOrganization[] = [ + { + id: 1, + organization_id: 10, + name: 'Ardeche', + is_enabled: true, + enabled_at: '2026-08-01T00:00:00Z', + }, +]; + +function cell(overrides: Partial): CoverageCell { + return { + organization_id: 10, + covered_date: '2026-08-03', + status: 'ok', + alerts_fetched: 0, + alerts_imported: 0, + alerts_skipped: 0, + alerts_failed: 0, + lanes_created: 0, + error: null, + ...overrides, + }; +} + +function renderHeatmap(cells: CoverageCell[]) { + return render( + + ); +} + +describe('CoverageHeatmap', () => { + it('renders one row per organization and one cell per day', () => { + renderHeatmap([]); + expect(screen.getByText('Ardeche')).toBeInTheDocument(); + expect(screen.getAllByRole('gridcell')).toHaveLength(4); + }); + + it('marks a day with imports as imported', () => { + renderHeatmap([ + cell({ covered_date: '2026-08-03', alerts_fetched: 5, alerts_imported: 5 }), + ]); + expect(screen.getByTestId('coverage-cell-10-2026-08-03')).toHaveAttribute( + 'data-state', + 'imported' + ); + }); + + it('distinguishes a covered day with zero alerts from a missing one', () => { + renderHeatmap([cell({ covered_date: '2026-08-03', alerts_imported: 0 })]); + expect(screen.getByTestId('coverage-cell-10-2026-08-03')).toHaveAttribute( + 'data-state', + 'empty' + ); + // No row was written for Aug 4 at all. + expect(screen.getByTestId('coverage-cell-10-2026-08-04')).toHaveAttribute( + 'data-state', + 'missing' + ); + }); + + it('marks a re-run day as imported even when the fresh run only skipped', () => { + // A re-run inside the trailing window re-fetches alerts already imported + // on a previous run and files them as skipped, not imported (see + // runner.py). The cell must still read as covered, not revert to empty. + renderHeatmap([ + cell({ + covered_date: '2026-08-03', + alerts_fetched: 5, + alerts_skipped: 5, + alerts_imported: 0, + }), + ]); + expect(screen.getByTestId('coverage-cell-10-2026-08-03')).toHaveAttribute( + 'data-state', + 'imported' + ); + }); + + it('marks failed days and surfaces the error in the tooltip', () => { + renderHeatmap([ + cell({ covered_date: '2026-08-03', status: 'failed', error: 'alert API down' }), + ]); + const target = screen.getByTestId('coverage-cell-10-2026-08-03'); + expect(target).toHaveAttribute('data-state', 'failed'); + expect(target.getAttribute('aria-label')).toContain('alert API down'); + }); + + it('marks partial days', () => { + renderHeatmap([ + cell({ + covered_date: '2026-08-03', + status: 'partial', + alerts_imported: 3, + alerts_failed: 2, + }), + ]); + expect(screen.getByTestId('coverage-cell-10-2026-08-03')).toHaveAttribute( + 'data-state', + 'partial' + ); + }); + + it('greys out days before the organization was enabled', () => { + render( + + ); + expect(screen.getByTestId('coverage-cell-10-2026-07-30')).toHaveAttribute( + 'data-state', + 'not-enabled' + ); + expect(screen.getByTestId('coverage-cell-10-2026-08-01')).toHaveAttribute( + 'data-state', + 'missing' + ); + }); + + it('renders a real coverage row before enabled_at instead of not-enabled', () => { + // The initial sweep imports the trailing window, which precedes + // enabled_at — those days have real coverage rows and must render their + // actual state, not be masked by the "not enabled yet" dash. + render( + + ); + expect(screen.getByTestId('coverage-cell-10-2026-07-30')).toHaveAttribute( + 'data-state', + 'imported' + ); + }); + + it('shows counts in the tooltip', () => { + renderHeatmap([ + cell({ + covered_date: '2026-08-03', + alerts_fetched: 9, + alerts_imported: 5, + alerts_skipped: 4, + }), + ]); + const label = screen + .getByTestId('coverage-cell-10-2026-08-03') + .getAttribute('aria-label'); + expect(label).toContain('5 imported'); + expect(label).toContain('4 skipped'); + }); + + it('shows a styled tooltip on hover and removes it on unhover', () => { + renderHeatmap([ + cell({ + covered_date: '2026-08-03', + alerts_fetched: 9, + alerts_imported: 5, + alerts_skipped: 4, + }), + ]); + expect(screen.queryByRole('tooltip')).toBeNull(); + fireEvent.mouseEnter(screen.getByTestId('coverage-cell-10-2026-08-03')); + expect(screen.getByRole('tooltip')).toHaveTextContent('5 imported'); + fireEvent.mouseLeave(screen.getByTestId('coverage-cell-10-2026-08-03')); + expect(screen.queryByRole('tooltip')).toBeNull(); + }); + + it('surfaces the error in the hover tooltip for failed cells', () => { + renderHeatmap([ + cell({ covered_date: '2026-08-03', status: 'failed', error: 'alert API down' }), + ]); + fireEvent.mouseEnter(screen.getByTestId('coverage-cell-10-2026-08-03')); + expect(screen.getByRole('tooltip')).toHaveTextContent('alert API down'); + }); + + it('renders covered-but-quiet days as pale pine, distinct from not-enabled', () => { + // User feedback 2026-08-07: the ash "covered, 0 alerts" square and the + // dashed "not enabled yet" outline were indistinguishable at 16px. A + // zero-alert covered day is a success state — it joins the green family. + renderHeatmap([cell({ covered_date: '2026-08-03', alerts_fetched: 0 })]); + const quiet = screen.getByTestId('coverage-cell-10-2026-08-03'); + expect(quiet).toHaveAttribute('data-state', 'empty'); + expect(quiet.className).toContain('bg-pine-soft'); + expect(quiet.className).not.toContain('bg-ash'); + }); +}); diff --git a/frontend/tests/hooks/useConnectors.test.tsx b/frontend/tests/hooks/useConnectors.test.tsx new file mode 100644 index 00000000..008bd48e --- /dev/null +++ b/frontend/tests/hooks/useConnectors.test.tsx @@ -0,0 +1,52 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest'; +import { renderHook, waitFor } from '@testing-library/react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import type { ReactNode } from 'react'; + +import { useConnectors, useToggleConnectorOrganization } from '@/hooks/useConnectors'; +import { apiClient } from '@/services/api'; + +vi.mock('@/services/api', () => ({ + apiClient: { + getConnectors: vi.fn(), + getConnectorOrganizations: vi.fn(), + toggleConnectorOrganization: vi.fn(), + }, +})); + +function wrapper({ children }: { children: ReactNode }) { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + return {children}; +} + +describe('useConnectors', () => { + beforeEach(() => vi.clearAllMocks()); + + it('returns the connector list', async () => { + vi.mocked(apiClient.getConnectors).mockResolvedValue([ + { id: 1, name: 'France', organizations_enabled: 2, organizations_total: 7 }, + ] as never); + + const { result } = renderHook(() => useConnectors(), { wrapper }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.data?.[0].name).toBe('France'); + }); +}); + +describe('useToggleConnectorOrganization', () => { + beforeEach(() => vi.clearAllMocks()); + + it('sends the connector id, organization id, and new state', async () => { + vi.mocked(apiClient.toggleConnectorOrganization).mockResolvedValue({} as never); + + const { result } = renderHook(() => useToggleConnectorOrganization(3), { wrapper }); + result.current.mutate({ organizationId: 42, isEnabled: true }); + + await waitFor(() => + expect(apiClient.toggleConnectorOrganization).toHaveBeenCalledWith(3, 42, true) + ); + }); +}); diff --git a/frontend/tests/pages/ConnectorDetailPage.test.tsx b/frontend/tests/pages/ConnectorDetailPage.test.tsx new file mode 100644 index 00000000..b9da4620 --- /dev/null +++ b/frontend/tests/pages/ConnectorDetailPage.test.tsx @@ -0,0 +1,158 @@ +import { describe, expect, it, vi, beforeEach } from 'vitest'; +import { render, screen, fireEvent } from '@testing-library/react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { MemoryRouter, Route, Routes } from 'react-router-dom'; + +import ConnectorDetailPage from '@/pages/ConnectorDetailPage'; +import { + useConnectors, + useConnectorOrganizations, + useConnectorCoverage, + useVerifyConnector, + useToggleConnectorOrganization, +} from '@/hooks/useConnectors'; + +const toggleMutate = vi.fn(); +const verifyMutate = vi.fn(); + +vi.mock('@/hooks/useConnectors', () => ({ + useConnectors: vi.fn(), + useConnectorOrganizations: vi.fn(), + useConnectorCoverage: vi.fn(), + useVerifyConnector: vi.fn(), + useToggleConnectorOrganization: vi.fn(), + useUpdateConnector: () => ({ mutate: vi.fn(), isPending: false }), +})); + +vi.mock('@/store/useAuthStore', () => ({ + useAuthStore: () => ({ + user: { id: 1, username: 'admin' }, + isSuperuser: () => true, + }), +})); + +const CONNECTOR = { + id: 1, + name: 'Pyronear France', + base_url: 'https://alertapi.pyronear.org', + source_api: 'pyronear_french', + login: 'admin', + has_password: true, + is_enabled: true, + trailing_days: 3, + image_transfer: null, + last_verified_at: '2026-08-06T03:00:00Z', + last_verify_error: null, + organizations_total: 2, + organizations_enabled: 1, +}; + +function renderPage() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + + } /> + + + + ); +} + +describe('ConnectorDetailPage', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(useConnectors).mockReturnValue({ + data: [CONNECTOR], + isLoading: false, + } as never); + vi.mocked(useConnectorOrganizations).mockReturnValue({ + data: [ + { + id: 1, + organization_id: 10, + name: 'Ardeche', + is_enabled: true, + enabled_at: '2026-08-01T00:00:00Z', + }, + { + id: 2, + organization_id: 20, + name: 'Aveyron', + is_enabled: false, + enabled_at: null, + }, + ], + isLoading: false, + } as never); + vi.mocked(useConnectorCoverage).mockReturnValue({ + data: [], + isLoading: false, + } as never); + vi.mocked(useVerifyConnector).mockReturnValue({ + mutate: verifyMutate, + isPending: false, + data: undefined, + } as never); + vi.mocked(useToggleConnectorOrganization).mockReturnValue({ + mutate: toggleMutate, + isPending: false, + } as never); + }); + + it('never renders the password, only that one is set', () => { + renderPage(); + expect(screen.getByText(/password is set/i)).toBeInTheDocument(); + expect(screen.queryByDisplayValue(/hunter/i)).not.toBeInTheDocument(); + }); + + it('lists discovered organizations with their enabled state', () => { + renderPage(); + expect(screen.getByLabelText('Ardeche')).toBeChecked(); + expect(screen.getByLabelText('Aveyron')).not.toBeChecked(); + }); + + it('toggling an organization calls the mutation', () => { + renderPage(); + fireEvent.click(screen.getByLabelText('Aveyron')); + expect(toggleMutate).toHaveBeenCalledWith({ organizationId: 20, isEnabled: true }); + }); + + it('triggers verification', () => { + renderPage(); + fireEvent.click(screen.getByRole('button', { name: /verify/i })); + expect(verifyMutate).toHaveBeenCalled(); + }); + + it('reports the cross-organization probe result after verifying', () => { + vi.mocked(useVerifyConnector).mockReturnValue({ + mutate: verifyMutate, + isPending: false, + data: { + ok: true, + error: null, + organizations: [], + organizations_seen_in_sample: 4, + organizations_total: 7, + sample_date: '2026-08-05', + }, + } as never); + + renderPage(); + expect(screen.getByText(/4 of 7/)).toBeInTheDocument(); + }); + + it('renders the coverage heatmap for enabled organizations only', () => { + renderPage(); + expect(screen.getAllByRole('gridcell').length).toBeGreaterThan(0); + // Aveyron is disabled, so it gets no heatmap row — but it still appears in + // the organization checkbox list above. + expect(screen.getAllByText('Ardeche').length).toBeGreaterThan(0); + // CoverageHeatmap tags each cell data-testid="coverage-cell-{organization_id}-{date}" + // (organization_id 20 is Aveyron). Scoped to that prefix, not to the name + // "Aveyron" itself, since Aveyron legitimately still appears in the + // checkbox list above. + expect(screen.queryAllByTestId(/^coverage-cell-20-/)).toHaveLength(0); + }); +}); diff --git a/frontend/tests/pages/ConnectorsPage.test.tsx b/frontend/tests/pages/ConnectorsPage.test.tsx new file mode 100644 index 00000000..ecbd5443 --- /dev/null +++ b/frontend/tests/pages/ConnectorsPage.test.tsx @@ -0,0 +1,193 @@ +import { fireEvent, render, screen } from '@testing-library/react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { MemoryRouter } from 'react-router-dom'; + +import ConnectorsPage from '@/pages/ConnectorsPage'; +import { useConnectors } from '@/hooks/useConnectors'; + +let testMutation: { + mutate: ReturnType; + reset: ReturnType; + isPending: boolean; + data: { ok: boolean; error: string | null; organizations_total: number } | undefined; + error: Error | null; +}; + +vi.mock('@/hooks/useConnectors', () => ({ + useConnectors: vi.fn(), + useCreateConnector: () => ({ mutate: vi.fn(), isPending: false }), + useDeleteConnector: () => ({ mutate: vi.fn(), isPending: false }), + useTestConnector: () => testMutation, +})); + +let isSuperuserValue = true; +vi.mock('@/store/useAuthStore', () => ({ + useAuthStore: () => ({ + user: { id: 1, username: 'admin' }, + isSuperuser: () => isSuperuserValue, + }), +})); + +function renderPage() { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + return render( + + + + + + ); +} + +describe('ConnectorsPage', () => { + beforeEach(() => { + vi.clearAllMocks(); + isSuperuserValue = true; + vi.mocked(useConnectors).mockReturnValue({ data: [], isLoading: false } as never); + }); + + it('refuses non-superusers', () => { + isSuperuserValue = false; + renderPage(); + expect(screen.getByText(/superuser privileges/i)).toBeInTheDocument(); + }); + + it('lists connectors with their enabled organization counts', () => { + vi.mocked(useConnectors).mockReturnValue({ + data: [ + { + id: 1, + name: 'Pyronear France', + base_url: 'https://alertapi.pyronear.org', + source_api: 'pyronear_french', + organizations_enabled: 3, + organizations_total: 7, + is_enabled: true, + last_verified_at: '2026-08-06T03:00:00Z', + last_verify_error: null, + }, + ], + isLoading: false, + } as never); + + renderPage(); + + expect(screen.getByText('Pyronear France')).toBeInTheDocument(); + expect(screen.getByText(/3 of 7/)).toBeInTheDocument(); + }); + + it('surfaces a verification error on the row', () => { + vi.mocked(useConnectors).mockReturnValue({ + data: [ + { + id: 1, + name: 'Broken', + base_url: 'https://x.example', + source_api: 'api_cenia', + organizations_enabled: 0, + organizations_total: 0, + is_enabled: true, + last_verified_at: null, + last_verify_error: 'RuntimeError: 401 Unauthorized', + }, + ], + isLoading: false, + } as never); + + renderPage(); + expect(screen.getByText(/401 Unauthorized/)).toBeInTheDocument(); + }); + + it('shows an empty state when there are no connectors', () => { + renderPage(); + expect(screen.getByText(/no connectors/i)).toBeInTheDocument(); + }); + + it('lets the table span the full layout width like the groups list', () => { + vi.mocked(useConnectors).mockReturnValue({ data: [], isLoading: false } as never); + const { container } = renderPage(); + // The width cap squeezed the 7-column table into horizontal scrolling; + // the reference table layout (SequenceGroupsListPage) has no cap. + expect(container.querySelector('.max-w-5xl')).toBeNull(); + }); +}); + +describe('create form — Test connection', () => { + beforeEach(() => { + testMutation = { + mutate: vi.fn(), + reset: vi.fn(), + isPending: false, + data: undefined, + error: null, + }; + vi.mocked(useConnectors).mockReturnValue({ data: [], isLoading: false } as never); + }); + + function openForm() { + renderPage(); + fireEvent.click(screen.getByRole('button', { name: /add connector/i })); + } + + function fillCredentials() { + fireEvent.change(screen.getByLabelText(/base url/i), { + target: { value: 'https://alertapi.pyronear.org' }, + }); + fireEvent.change(screen.getByLabelText(/^login$/i), { target: { value: 'admin' } }); + fireEvent.change(screen.getByLabelText(/^password$/i), { target: { value: 's3cret' } }); + } + + it('disables the button until base URL, login, and password are filled', () => { + openForm(); + const button = screen.getByRole('button', { name: /test connection/i }); + expect(button).toBeDisabled(); + fillCredentials(); + expect(button).toBeEnabled(); + }); + + it('sends exactly the three credential fields', () => { + openForm(); + fillCredentials(); + fireEvent.click(screen.getByRole('button', { name: /test connection/i })); + expect(testMutation.mutate).toHaveBeenCalledWith({ + base_url: 'https://alertapi.pyronear.org', + login: 'admin', + password: 's3cret', + }); + }); + + it('shows a pending state while the test runs', () => { + testMutation.isPending = true; + openForm(); + const button = screen.getByRole('button', { name: /testing/i }); + expect(button).toBeDisabled(); + }); + + it('renders a success result with the organization count', () => { + testMutation.data = { ok: true, error: null, organizations_total: 21 }; + openForm(); + expect( + screen.getByText(/connection ok — 21 organizations visible/i) + ).toBeInTheDocument(); + }); + + it('renders the backend error verbatim on failure', () => { + testMutation.data = { + ok: false, + error: + 'ValueError: alert API returned an unexpected organizations response: Incompatible token scope.', + organizations_total: 0, + }; + openForm(); + expect(screen.getByText(/incompatible token scope/i)).toBeInTheDocument(); + }); + + it('clears a previous result when a credential field changes', () => { + testMutation.data = { ok: true, error: null, organizations_total: 21 }; + openForm(); + fireEvent.change(screen.getByLabelText(/^password$/i), { + target: { value: 'different' }, + }); + expect(testMutation.reset).toHaveBeenCalled(); + }); +}); diff --git a/frontend/tests/pages/UserManagementPage.test.tsx b/frontend/tests/pages/UserManagementPage.test.tsx index 9fbf5d9b..dc85ee80 100644 --- a/frontend/tests/pages/UserManagementPage.test.tsx +++ b/frontend/tests/pages/UserManagementPage.test.tsx @@ -33,6 +33,26 @@ const localizer: User = { created_at: '2026-01-02T00:00:00Z', }; +const inactiveUser: User = { + id: 3, + username: 'benched', + is_active: false, + is_superuser: false, + can_localize: false, + is_system: false, + created_at: '2026-01-02T00:00:00Z', +}; + +const systemUser: User = { + id: 99, + username: 'worker', + is_active: true, + is_superuser: false, + can_localize: false, + is_system: true, + created_at: '2026-01-02T00:00:00Z', +}; + const renderPage = () => { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } }, @@ -147,3 +167,44 @@ describe('UserManagementPage can_localize', () => { ); }); }); + +describe('UserManagementPage status pill', () => { + it('shows a neutral Service pill for system users, not Active', async () => { + vi.mocked(apiClient.getUsers).mockResolvedValue({ + items: [localizer, systemUser], + page: 1, + pages: 1, + size: 50, + total: 2, + }); + + renderPage(); + + const systemRow = (await screen.findByText('worker')).closest('tr') as HTMLElement; + expect(within(systemRow).getByText('Service')).toBeInTheDocument(); + expect(within(systemRow).queryByText('Active')).not.toBeInTheDocument(); + expect(within(systemRow).queryByText('Inactive')).not.toBeInTheDocument(); + }); + + it('still shows Active for an ordinary active user', async () => { + renderPage(); + + const row = (await screen.findByText('scout')).closest('tr') as HTMLElement; + expect(within(row).getByText('Active')).toBeInTheDocument(); + }); + + it('still shows Inactive for an ordinary inactive user', async () => { + vi.mocked(apiClient.getUsers).mockResolvedValue({ + items: [inactiveUser], + page: 1, + pages: 1, + size: 50, + total: 1, + }); + + renderPage(); + + const row = (await screen.findByText('benched')).closest('tr') as HTMLElement; + expect(within(row).getByText('Inactive')).toBeInTheDocument(); + }); +});