From 7b04b72936cdbbb02291136aa16cdc0551341ca6 Mon Sep 17 00:00:00 2001 From: Chouffe Date: Thu, 13 Aug 2026 17:01:06 +0200 Subject: [PATCH 1/2] feat(export): carry the alert's temporal model score Ranking exported objects by the platform's temporal-model verdict is the entry point for hard data mining: high score + false_positive lane is a hard negative, low score + smoke lane a hard positive. The score sits at alert level rather than on each object because that is its true grain -- the platform scores a platform sequence, and our object-splitting is annotator-side. The verdict rides the primary lane while every split sibling stays NULL, so max() over the alert's lanes collapses the group losslessly. It deliberately spans ALL lanes rather than only exported ones, so an unsure primary lane cannot erase its alert's score. Objects still rank fine: each shares a manifest line with its alert. null means never scored, never 'scored low' -- 0.0 is a real verdict, so consumers must drop nulls rather than coalesce them. Scores only exist for alerts imported from 2026-08-10 on; the backfill is still deferred. The pull script needs no change: it copies the API payload verbatim, so the fields reach manifest.jsonl on their own. --- CLAUDE.md | 2 + .../src/app/api/api_v1/endpoints/export.py | 21 +++ .../src/tests/endpoints/test_export.py | 156 ++++++++++++++++++ 3 files changed, 179 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 629b97d4..3ad35305 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,6 +98,8 @@ make export-alerts OUTPUT_DIR=outputs/alerts_export Writes `manifest.jsonl` (one line per alert) plus `images/{source_api}/{platform_alert_id}/{detection_id}.jpg`; each frame carries an `image_path` into that tree. Only alerts whose every lane reached `ANNOTATED` are exported. Re-runs are idempotent — the manifest is rewritten and only missing images are downloaded. +Each alert also carries `temporal_model_score` (plus `temporal_model_version` / `temporal_api_version`) for score-based mining: rank objects by their alert's score and filter on `record_kind` to surface hard negatives. The score is alert-level because the platform's temporal model scores an alert, not an object — the verdict rides the primary lane and object-split siblings hold NULL. `null` means **never scored** (alerts imported before 2026-08-10, fail-opens, risk-gated sequences), never "scored low", so drop nulls when ranking rather than coalescing them to `0.0`; `0.0` itself is a real verdict. + ### Export QA overlays ```bash diff --git a/annotation_api/src/app/api/api_v1/endpoints/export.py b/annotation_api/src/app/api/api_v1/endpoints/export.py index 515288fb..31dd9f7d 100644 --- a/annotation_api/src/app/api/api_v1/endpoints/export.py +++ b/annotation_api/src/app/api/api_v1/endpoints/export.py @@ -64,6 +64,13 @@ class AlertExportItem(BaseModel): azimuth: Optional[int] = None recorded_at: datetime last_annotated_at: datetime + # Platform temporal-model verdict for the alert. NULL means the platform + # never scored it (pre-2026-06-11 alerts, fail-opens, anything imported + # before the column existed); it never means "scored low", so consumers + # ranking on it must drop nulls rather than coalesce them to 0.0. + temporal_model_score: Optional[float] = None + temporal_model_version: Optional[str] = None + temporal_api_version: Optional[str] = None objects: List[ObjectExport] @@ -205,6 +212,14 @@ async def export_alerts( # Alert start deliberately spans ALL lanes, unsure ones included — the # alert began when its first object appeared, exported or not. alert_recorded_at = func.min(Sequence.recorded_at) + # The platform scores an alert, not an object: the verdict rides the + # primary lane and every object-split sibling stays NULL by import + # construction, so max() collapses the group without losing anything. + # Spans ALL lanes like alert_recorded_at — an unsure primary lane must + # not erase its alert's score. + temporal_model_score = func.max(Sequence.temporal_model_score) + temporal_model_version = func.max(Sequence.temporal_model_version) + temporal_api_version = func.max(Sequence.temporal_api_version) stmt = ( select( @@ -212,6 +227,9 @@ async def export_alerts( Sequence.platform_alert_id.label("platform_alert_id"), alert_recorded_at.label("recorded_at"), last_annotated_at.label("last_annotated_at"), + temporal_model_score.label("temporal_model_score"), + temporal_model_version.label("temporal_model_version"), + temporal_api_version.label("temporal_api_version"), ) .select_from(Sequence) .outerjoin(SequenceAnnotation, SequenceAnnotation.sequence_id == Sequence.id) @@ -394,6 +412,9 @@ async def export_alerts( azimuth=first_seq.azimuth, recorded_at=row.recorded_at, last_annotated_at=row.last_annotated_at, + temporal_model_score=row.temporal_model_score, + temporal_model_version=row.temporal_model_version, + temporal_api_version=row.temporal_api_version, objects=objects, ) ) diff --git a/annotation_api/src/tests/endpoints/test_export.py b/annotation_api/src/tests/endpoints/test_export.py index cba953d0..edffacd4 100644 --- a/annotation_api/src/tests/endpoints/test_export.py +++ b/annotation_api/src/tests/endpoints/test_export.py @@ -58,6 +58,9 @@ async def create_lane( organisation_name: str = "Export Org", organisation_id: int = 70, recorded_at: Optional[datetime] = None, + temporal_model_score: Optional[float] = None, + temporal_model_version: Optional[str] = None, + temporal_api_version: Optional[str] = None, ) -> int: """Create one sequence (lane) of an alert, returns sequence id.""" payload = { @@ -73,6 +76,14 @@ async def create_lane( "recorded_at": (recorded_at or now).isoformat(), "last_seen_at": (recorded_at or now).isoformat(), } + # Omitted rather than sent empty: the platform scores an alert, so only + # its primary lane carries these; siblings must stay NULL. + if temporal_model_score is not None: + payload["temporal_model_score"] = str(temporal_model_score) + if temporal_model_version is not None: + payload["temporal_model_version"] = temporal_model_version + if temporal_api_version is not None: + payload["temporal_api_version"] = temporal_api_version resp = await client.post("/sequences", data=payload) assert resp.status_code == 201, resp.text return resp.json()["id"] @@ -874,6 +885,151 @@ async def test_export_alerts_annotation_updated_watermark( assert [i["platform_alert_id"] for i in items] == [7602] +@pytest.mark.asyncio +async def test_export_alerts_carries_alert_temporal_score( + authenticated_client: AsyncClient, + sequence_session, + detection_session, + dummy_bucket, +): + """The platform's temporal-model verdict is an ALERT property: it rides + the scored primary lane, and the export surfaces it once at alert level + even though the sibling lane holds NULL.""" + scored_seq = await create_lane( + authenticated_client, + platform_alert_id=7801, + alert_api_id=7801, + temporal_model_score=0.87, + temporal_model_version="0.2.0", + temporal_api_version="0.3.1", + ) + sibling_seq = await create_lane( + authenticated_client, platform_alert_id=7801, alert_api_id=1000007801001 + ) + for seq_id in (scored_seq, sibling_seq): + det_id = await create_frame( + authenticated_client, sequence_id=seq_id, alert_api_id=1 + ) + await annotate_lane( + authenticated_client, + sequence_id=seq_id, + detection_ids=[det_id], + is_smoke=False, + false_positive_types=["antenna"], + ) + + resp = await authenticated_client.get("/export/alerts") + assert resp.status_code == 200, resp.text + items = resp.json()["items"] + assert len(items) == 1 + alert = items[0] + assert alert["temporal_model_score"] == 0.87 + assert alert["temporal_model_version"] == "0.2.0" + assert alert["temporal_api_version"] == "0.3.1" + + +@pytest.mark.asyncio +async def test_export_alerts_temporal_score_null_when_never_scored( + authenticated_client: AsyncClient, + sequence_session, + detection_session, + dummy_bucket, +): + """An alert the platform never scored exports null, never 0.0 — the + difference is 'no verdict' vs 'confidently not smoke'.""" + await seed_minimal_fp_alert(authenticated_client, platform_alert_id=7802) + + resp = await authenticated_client.get("/export/alerts") + alert = resp.json()["items"][0] + assert alert["temporal_model_score"] is None + assert alert["temporal_model_version"] is None + assert alert["temporal_api_version"] is None + + +@pytest.mark.asyncio +async def test_export_alerts_temporal_score_zero_survives( + authenticated_client: AsyncClient, + sequence_session, + detection_session, + dummy_bucket, +): + """0.0 is a real production verdict and must not be flattened to null by + any falsy-value handling on the way out.""" + seq_id = await create_lane( + authenticated_client, + platform_alert_id=7803, + alert_api_id=7803, + temporal_model_score=0.0, + temporal_model_version="0.2.0", + ) + det_id = await create_frame( + authenticated_client, sequence_id=seq_id, alert_api_id=1 + ) + await annotate_lane( + authenticated_client, + sequence_id=seq_id, + detection_ids=[det_id], + is_smoke=False, + false_positive_types=["antenna"], + ) + + resp = await authenticated_client.get("/export/alerts") + alert = resp.json()["items"][0] + assert alert["temporal_model_score"] == 0.0 + assert alert["temporal_model_score"] is not None + + +@pytest.mark.asyncio +async def test_export_alerts_temporal_score_survives_unsure_scored_lane( + authenticated_client: AsyncClient, + sequence_session, + detection_session, + dummy_bucket, +): + """The score aggregates over ALL lanes, not just exported ones: when the + scored lane is unsure (and so omitted from objects), the alert it belongs + to must still report the platform's verdict.""" + unsure_scored_seq = await create_lane( + authenticated_client, + platform_alert_id=7804, + alert_api_id=7804, + temporal_model_score=0.42, + temporal_model_version="0.2.0", + ) + sure_seq = await create_lane( + authenticated_client, platform_alert_id=7804, alert_api_id=1000007804001 + ) + unsure_det = await create_frame( + authenticated_client, sequence_id=unsure_scored_seq, alert_api_id=1 + ) + sure_det = await create_frame( + authenticated_client, sequence_id=sure_seq, alert_api_id=1 + ) + await annotate_lane( + authenticated_client, + sequence_id=unsure_scored_seq, + detection_ids=[unsure_det], + is_smoke=True, + smoke_type="wildfire", + is_unsure=True, + ) + await annotate_lane( + authenticated_client, + sequence_id=sure_seq, + detection_ids=[sure_det], + is_smoke=False, + false_positive_types=["antenna"], + ) + + resp = await authenticated_client.get("/export/alerts") + items = resp.json()["items"] + assert len(items) == 1 + alert = items[0] + assert [o["sequence_id"] for o in alert["objects"]] == [sure_seq] + assert alert["temporal_model_score"] == 0.42 + assert alert["temporal_model_version"] == "0.2.0" + + @pytest.mark.asyncio async def test_export_alerts_requires_auth(async_client: AsyncClient): resp = await async_client.get("/export/alerts") From 1c27967b2a0ea8662ffefe2f9180968cd00ae68e Mon Sep 17 00:00:00 2001 From: Chouffe Date: Thu, 13 Aug 2026 17:20:07 +0200 Subject: [PATCH 2/2] docs(export): correct the null contract and the watermark's blind spot Review of #382 surfaced two ways the documented semantics were wrong. null does not only mean 'never scored'. When the object split cannot tell which lane the platform scored, object_split.py clears the temporal keys on EVERY lane, primary included, so a genuinely scored alert exports null. A miner following the old wording would discard a real verdict as 'no verdict'. The incremental-sync watermark does not cover scores. PATCH /sequences/temporal-score writes only the Sequence columns and touches no annotation row, so last_annotated_at never moves. The deferred backfill would be invisible to any consumer syncing on annotation_updated_gte; it needs a full pull. Also refreshes the export spec, whose response example and alert-level field list enumerate the payload exhaustively and had gone stale. --- CLAUDE.md | 2 +- .../src/app/api/api_v1/endpoints/export.py | 18 +++++++++++++----- ...2026-08-06-export-alerts-endpoint-design.md | 15 +++++++++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3ad35305..cea0d1ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,7 +98,7 @@ make export-alerts OUTPUT_DIR=outputs/alerts_export Writes `manifest.jsonl` (one line per alert) plus `images/{source_api}/{platform_alert_id}/{detection_id}.jpg`; each frame carries an `image_path` into that tree. Only alerts whose every lane reached `ANNOTATED` are exported. Re-runs are idempotent — the manifest is rewritten and only missing images are downloaded. -Each alert also carries `temporal_model_score` (plus `temporal_model_version` / `temporal_api_version`) for score-based mining: rank objects by their alert's score and filter on `record_kind` to surface hard negatives. The score is alert-level because the platform's temporal model scores an alert, not an object — the verdict rides the primary lane and object-split siblings hold NULL. `null` means **never scored** (alerts imported before 2026-08-10, fail-opens, risk-gated sequences), never "scored low", so drop nulls when ranking rather than coalescing them to `0.0`; `0.0` itself is a real verdict. +Each alert also carries `temporal_model_score` (plus `temporal_model_version` / `temporal_api_version`) for score-based mining: rank objects by their alert's score and filter on `record_kind` to surface hard negatives. The score is alert-level because the platform's temporal model scores an alert, not an object — the verdict rides the primary lane and object-split siblings hold NULL. `null` means **no verdict is attributed**: either never scored (alerts imported before 2026-08-10, fail-opens, risk-gated sequences) or scored but not attributable to a lane during the object split. It never means "scored low", so drop nulls when ranking rather than coalescing them to `0.0`; `0.0` itself is a real verdict. Note a score refresh does not move `last_annotated_at`, so a backfill only shows up in a full pull, not an `annotation_updated_gte` one. ### Export QA overlays diff --git a/annotation_api/src/app/api/api_v1/endpoints/export.py b/annotation_api/src/app/api/api_v1/endpoints/export.py index 31dd9f7d..293fdb09 100644 --- a/annotation_api/src/app/api/api_v1/endpoints/export.py +++ b/annotation_api/src/app/api/api_v1/endpoints/export.py @@ -64,10 +64,15 @@ class AlertExportItem(BaseModel): azimuth: Optional[int] = None recorded_at: datetime last_annotated_at: datetime - # Platform temporal-model verdict for the alert. NULL means the platform - # never scored it (pre-2026-06-11 alerts, fail-opens, anything imported - # before the column existed); it never means "scored low", so consumers - # ranking on it must drop nulls rather than coalesce them to 0.0. + # Platform temporal-model verdict for the alert. NULL means no verdict is + # attributed to this alert, which covers two different situations: the + # platform never scored it (pre-2026-06-11 alerts, fail-opens, risk-gated + # or sub-MIN_FRAMES sequences, anything imported before the column + # existed), OR it was scored but the object split could not tell which + # lane the score belonged to and cleared it from all of them + # (object_split.py, `primary_identified` false). NULL never means "scored + # low", so consumers ranking on it must drop nulls rather than coalesce + # them to 0.0 — and must not read a null as evidence of a low verdict. temporal_model_score: Optional[float] = None temporal_model_version: Optional[str] = None temporal_api_version: Optional[str] = None @@ -163,7 +168,10 @@ async def export_alerts( None, description=( "Incremental-sync watermark: alerts whose last_annotated_at is " - "greater or equal to this date" + "greater or equal to this date. Covers annotation work only — a " + "temporal-score refresh writes no annotation row, so scores " + "backfilled onto already-annotated alerts do NOT move this " + "watermark and need a full pull to appear." ), ), smoke_types: Optional[List[SmokeType]] = Query( diff --git a/docs/specs/2026-08-06-export-alerts-endpoint-design.md b/docs/specs/2026-08-06-export-alerts-endpoint-design.md index 055302eb..0f3e7f34 100644 --- a/docs/specs/2026-08-06-export-alerts-endpoint-design.md +++ b/docs/specs/2026-08-06-export-alerts-endpoint-design.md @@ -69,6 +69,9 @@ overlay row) restores the alert to the export untouched. "azimuth": 200, "recorded_at": "2026-07-14T15:42:10", "last_annotated_at": "2026-07-16T10:03:17", + "temporal_model_score": 0.87, + "temporal_model_version": "0.2.0", + "temporal_api_version": "0.3.1", "objects": [ { "sequence_id": 3121, @@ -136,6 +139,18 @@ azimuth, `recorded_at` (alert start, min over lanes' `recorded_at`), and annotations) across all exported lanes. `last_annotated_at` is the value the `annotation_updated_gte` filter compares against. +Also alert level: `temporal_model_score` and its provenance pair +`temporal_model_version` / `temporal_api_version`, each a `max` over the +alert's lanes. Alert grain is the score's true grain — the platform scores a +platform sequence, and object-splitting is annotator-side, so the verdict +rides the primary lane while split siblings stay NULL and `max` collapses the +group losslessly. Unlike `last_annotated_at`, these span **all** lanes rather +than only exported ones, so an unsure primary lane cannot erase its alert's +score. `null` means no verdict is attributed (never scored, or scored but not +attributable to a lane during the split) — never "scored low"; `0.0` is a real +verdict. A temporal-score refresh does not move `last_annotated_at`, so +backfilled scores require a full pull rather than an incremental one. + **Object level:** - `sequence_id` is the lane/track identity. The importer's synthetic `alert_api_id`