diff --git a/CLAUDE.md b/CLAUDE.md index 629b97d4..cea0d1ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,6 +98,8 @@ make export-alerts OUTPUT_DIR=outputs/alerts_export Writes `manifest.jsonl` (one line per alert) plus `images/{source_api}/{platform_alert_id}/{detection_id}.jpg`; each frame carries an `image_path` into that tree. Only alerts whose every lane reached `ANNOTATED` are exported. Re-runs are idempotent — the manifest is rewritten and only missing images are downloaded. +Each alert also carries `temporal_model_score` (plus `temporal_model_version` / `temporal_api_version`) for score-based mining: rank objects by their alert's score and filter on `record_kind` to surface hard negatives. The score is alert-level because the platform's temporal model scores an alert, not an object — the verdict rides the primary lane and object-split siblings hold NULL. `null` means **no verdict is attributed**: either never scored (alerts imported before 2026-08-10, fail-opens, risk-gated sequences) or scored but not attributable to a lane during the object split. It never means "scored low", so drop nulls when ranking rather than coalescing them to `0.0`; `0.0` itself is a real verdict. Note a score refresh does not move `last_annotated_at`, so a backfill only shows up in a full pull, not an `annotation_updated_gte` one. + ### Export QA overlays ```bash diff --git a/annotation_api/src/app/api/api_v1/endpoints/export.py b/annotation_api/src/app/api/api_v1/endpoints/export.py index 515288fb..293fdb09 100644 --- a/annotation_api/src/app/api/api_v1/endpoints/export.py +++ b/annotation_api/src/app/api/api_v1/endpoints/export.py @@ -64,6 +64,18 @@ class AlertExportItem(BaseModel): azimuth: Optional[int] = None recorded_at: datetime last_annotated_at: datetime + # Platform temporal-model verdict for the alert. NULL means no verdict is + # attributed to this alert, which covers two different situations: the + # platform never scored it (pre-2026-06-11 alerts, fail-opens, risk-gated + # or sub-MIN_FRAMES sequences, anything imported before the column + # existed), OR it was scored but the object split could not tell which + # lane the score belonged to and cleared it from all of them + # (object_split.py, `primary_identified` false). NULL never means "scored + # low", so consumers ranking on it must drop nulls rather than coalesce + # them to 0.0 — and must not read a null as evidence of a low verdict. + temporal_model_score: Optional[float] = None + temporal_model_version: Optional[str] = None + temporal_api_version: Optional[str] = None objects: List[ObjectExport] @@ -156,7 +168,10 @@ async def export_alerts( None, description=( "Incremental-sync watermark: alerts whose last_annotated_at is " - "greater or equal to this date" + "greater or equal to this date. Covers annotation work only — a " + "temporal-score refresh writes no annotation row, so scores " + "backfilled onto already-annotated alerts do NOT move this " + "watermark and need a full pull to appear." ), ), smoke_types: Optional[List[SmokeType]] = Query( @@ -205,6 +220,14 @@ async def export_alerts( # Alert start deliberately spans ALL lanes, unsure ones included — the # alert began when its first object appeared, exported or not. alert_recorded_at = func.min(Sequence.recorded_at) + # The platform scores an alert, not an object: the verdict rides the + # primary lane and every object-split sibling stays NULL by import + # construction, so max() collapses the group without losing anything. + # Spans ALL lanes like alert_recorded_at — an unsure primary lane must + # not erase its alert's score. + temporal_model_score = func.max(Sequence.temporal_model_score) + temporal_model_version = func.max(Sequence.temporal_model_version) + temporal_api_version = func.max(Sequence.temporal_api_version) stmt = ( select( @@ -212,6 +235,9 @@ async def export_alerts( Sequence.platform_alert_id.label("platform_alert_id"), alert_recorded_at.label("recorded_at"), last_annotated_at.label("last_annotated_at"), + temporal_model_score.label("temporal_model_score"), + temporal_model_version.label("temporal_model_version"), + temporal_api_version.label("temporal_api_version"), ) .select_from(Sequence) .outerjoin(SequenceAnnotation, SequenceAnnotation.sequence_id == Sequence.id) @@ -394,6 +420,9 @@ async def export_alerts( azimuth=first_seq.azimuth, recorded_at=row.recorded_at, last_annotated_at=row.last_annotated_at, + temporal_model_score=row.temporal_model_score, + temporal_model_version=row.temporal_model_version, + temporal_api_version=row.temporal_api_version, objects=objects, ) ) diff --git a/annotation_api/src/tests/endpoints/test_export.py b/annotation_api/src/tests/endpoints/test_export.py index cba953d0..edffacd4 100644 --- a/annotation_api/src/tests/endpoints/test_export.py +++ b/annotation_api/src/tests/endpoints/test_export.py @@ -58,6 +58,9 @@ async def create_lane( organisation_name: str = "Export Org", organisation_id: int = 70, recorded_at: Optional[datetime] = None, + temporal_model_score: Optional[float] = None, + temporal_model_version: Optional[str] = None, + temporal_api_version: Optional[str] = None, ) -> int: """Create one sequence (lane) of an alert, returns sequence id.""" payload = { @@ -73,6 +76,14 @@ async def create_lane( "recorded_at": (recorded_at or now).isoformat(), "last_seen_at": (recorded_at or now).isoformat(), } + # Omitted rather than sent empty: the platform scores an alert, so only + # its primary lane carries these; siblings must stay NULL. + if temporal_model_score is not None: + payload["temporal_model_score"] = str(temporal_model_score) + if temporal_model_version is not None: + payload["temporal_model_version"] = temporal_model_version + if temporal_api_version is not None: + payload["temporal_api_version"] = temporal_api_version resp = await client.post("/sequences", data=payload) assert resp.status_code == 201, resp.text return resp.json()["id"] @@ -874,6 +885,151 @@ async def test_export_alerts_annotation_updated_watermark( assert [i["platform_alert_id"] for i in items] == [7602] +@pytest.mark.asyncio +async def test_export_alerts_carries_alert_temporal_score( + authenticated_client: AsyncClient, + sequence_session, + detection_session, + dummy_bucket, +): + """The platform's temporal-model verdict is an ALERT property: it rides + the scored primary lane, and the export surfaces it once at alert level + even though the sibling lane holds NULL.""" + scored_seq = await create_lane( + authenticated_client, + platform_alert_id=7801, + alert_api_id=7801, + temporal_model_score=0.87, + temporal_model_version="0.2.0", + temporal_api_version="0.3.1", + ) + sibling_seq = await create_lane( + authenticated_client, platform_alert_id=7801, alert_api_id=1000007801001 + ) + for seq_id in (scored_seq, sibling_seq): + det_id = await create_frame( + authenticated_client, sequence_id=seq_id, alert_api_id=1 + ) + await annotate_lane( + authenticated_client, + sequence_id=seq_id, + detection_ids=[det_id], + is_smoke=False, + false_positive_types=["antenna"], + ) + + resp = await authenticated_client.get("/export/alerts") + assert resp.status_code == 200, resp.text + items = resp.json()["items"] + assert len(items) == 1 + alert = items[0] + assert alert["temporal_model_score"] == 0.87 + assert alert["temporal_model_version"] == "0.2.0" + assert alert["temporal_api_version"] == "0.3.1" + + +@pytest.mark.asyncio +async def test_export_alerts_temporal_score_null_when_never_scored( + authenticated_client: AsyncClient, + sequence_session, + detection_session, + dummy_bucket, +): + """An alert the platform never scored exports null, never 0.0 — the + difference is 'no verdict' vs 'confidently not smoke'.""" + await seed_minimal_fp_alert(authenticated_client, platform_alert_id=7802) + + resp = await authenticated_client.get("/export/alerts") + alert = resp.json()["items"][0] + assert alert["temporal_model_score"] is None + assert alert["temporal_model_version"] is None + assert alert["temporal_api_version"] is None + + +@pytest.mark.asyncio +async def test_export_alerts_temporal_score_zero_survives( + authenticated_client: AsyncClient, + sequence_session, + detection_session, + dummy_bucket, +): + """0.0 is a real production verdict and must not be flattened to null by + any falsy-value handling on the way out.""" + seq_id = await create_lane( + authenticated_client, + platform_alert_id=7803, + alert_api_id=7803, + temporal_model_score=0.0, + temporal_model_version="0.2.0", + ) + det_id = await create_frame( + authenticated_client, sequence_id=seq_id, alert_api_id=1 + ) + await annotate_lane( + authenticated_client, + sequence_id=seq_id, + detection_ids=[det_id], + is_smoke=False, + false_positive_types=["antenna"], + ) + + resp = await authenticated_client.get("/export/alerts") + alert = resp.json()["items"][0] + assert alert["temporal_model_score"] == 0.0 + assert alert["temporal_model_score"] is not None + + +@pytest.mark.asyncio +async def test_export_alerts_temporal_score_survives_unsure_scored_lane( + authenticated_client: AsyncClient, + sequence_session, + detection_session, + dummy_bucket, +): + """The score aggregates over ALL lanes, not just exported ones: when the + scored lane is unsure (and so omitted from objects), the alert it belongs + to must still report the platform's verdict.""" + unsure_scored_seq = await create_lane( + authenticated_client, + platform_alert_id=7804, + alert_api_id=7804, + temporal_model_score=0.42, + temporal_model_version="0.2.0", + ) + sure_seq = await create_lane( + authenticated_client, platform_alert_id=7804, alert_api_id=1000007804001 + ) + unsure_det = await create_frame( + authenticated_client, sequence_id=unsure_scored_seq, alert_api_id=1 + ) + sure_det = await create_frame( + authenticated_client, sequence_id=sure_seq, alert_api_id=1 + ) + await annotate_lane( + authenticated_client, + sequence_id=unsure_scored_seq, + detection_ids=[unsure_det], + is_smoke=True, + smoke_type="wildfire", + is_unsure=True, + ) + await annotate_lane( + authenticated_client, + sequence_id=sure_seq, + detection_ids=[sure_det], + is_smoke=False, + false_positive_types=["antenna"], + ) + + resp = await authenticated_client.get("/export/alerts") + items = resp.json()["items"] + assert len(items) == 1 + alert = items[0] + assert [o["sequence_id"] for o in alert["objects"]] == [sure_seq] + assert alert["temporal_model_score"] == 0.42 + assert alert["temporal_model_version"] == "0.2.0" + + @pytest.mark.asyncio async def test_export_alerts_requires_auth(async_client: AsyncClient): resp = await async_client.get("/export/alerts") diff --git a/docs/specs/2026-08-06-export-alerts-endpoint-design.md b/docs/specs/2026-08-06-export-alerts-endpoint-design.md index 055302eb..0f3e7f34 100644 --- a/docs/specs/2026-08-06-export-alerts-endpoint-design.md +++ b/docs/specs/2026-08-06-export-alerts-endpoint-design.md @@ -69,6 +69,9 @@ overlay row) restores the alert to the export untouched. "azimuth": 200, "recorded_at": "2026-07-14T15:42:10", "last_annotated_at": "2026-07-16T10:03:17", + "temporal_model_score": 0.87, + "temporal_model_version": "0.2.0", + "temporal_api_version": "0.3.1", "objects": [ { "sequence_id": 3121, @@ -136,6 +139,18 @@ azimuth, `recorded_at` (alert start, min over lanes' `recorded_at`), and annotations) across all exported lanes. `last_annotated_at` is the value the `annotation_updated_gte` filter compares against. +Also alert level: `temporal_model_score` and its provenance pair +`temporal_model_version` / `temporal_api_version`, each a `max` over the +alert's lanes. Alert grain is the score's true grain — the platform scores a +platform sequence, and object-splitting is annotator-side, so the verdict +rides the primary lane while split siblings stay NULL and `max` collapses the +group losslessly. Unlike `last_annotated_at`, these span **all** lanes rather +than only exported ones, so an unsure primary lane cannot erase its alert's +score. `null` means no verdict is attributed (never scored, or scored but not +attributable to a lane during the split) — never "scored low"; `0.0` is a real +verdict. A temporal-score refresh does not move `last_annotated_at`, so +backfilled scores require a full pull rather than an incremental one. + **Object level:** - `sequence_id` is the lane/track identity. The importer's synthetic `alert_api_id`