Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ make export-alerts OUTPUT_DIR=outputs/alerts_export

Writes `manifest.jsonl` (one line per alert) plus `images/{source_api}/{platform_alert_id}/{detection_id}.jpg`; each frame carries an `image_path` into that tree. Only alerts whose every lane reached `ANNOTATED` are exported. Re-runs are idempotent — the manifest is rewritten and only missing images are downloaded.

Each alert also carries `temporal_model_score` (plus `temporal_model_version` / `temporal_api_version`) for score-based mining: rank objects by their alert's score and filter on `record_kind` to surface hard negatives. The score is alert-level because the platform's temporal model scores an alert, not an object — the verdict rides the primary lane and object-split siblings hold NULL. `null` means **no verdict is attributed**: either never scored (alerts imported before 2026-08-10, fail-opens, risk-gated sequences) or scored but not attributable to a lane during the object split. It never means "scored low", so drop nulls when ranking rather than coalescing them to `0.0`; `0.0` itself is a real verdict. Note a score refresh does not move `last_annotated_at`, so a backfill only shows up in a full pull, not an `annotation_updated_gte` one.

### Export QA overlays

```bash
Expand Down
31 changes: 30 additions & 1 deletion annotation_api/src/app/api/api_v1/endpoints/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,18 @@ class AlertExportItem(BaseModel):
azimuth: Optional[int] = None
recorded_at: datetime
last_annotated_at: datetime
# Platform temporal-model verdict for the alert. NULL means no verdict is
# attributed to this alert, which covers two different situations: the
# platform never scored it (pre-2026-06-11 alerts, fail-opens, risk-gated
# or sub-MIN_FRAMES sequences, anything imported before the column
# existed), OR it was scored but the object split could not tell which
# lane the score belonged to and cleared it from all of them
# (object_split.py, `primary_identified` false). NULL never means "scored
# low", so consumers ranking on it must drop nulls rather than coalesce
# them to 0.0 — and must not read a null as evidence of a low verdict.
temporal_model_score: Optional[float] = None
temporal_model_version: Optional[str] = None
temporal_api_version: Optional[str] = None
objects: List[ObjectExport]


Expand Down Expand Up @@ -156,7 +168,10 @@ async def export_alerts(
None,
description=(
"Incremental-sync watermark: alerts whose last_annotated_at is "
"greater or equal to this date"
"greater or equal to this date. Covers annotation work only — a "
"temporal-score refresh writes no annotation row, so scores "
"backfilled onto already-annotated alerts do NOT move this "
"watermark and need a full pull to appear."
),
),
smoke_types: Optional[List[SmokeType]] = Query(
Expand Down Expand Up @@ -205,13 +220,24 @@ async def export_alerts(
# Alert start deliberately spans ALL lanes, unsure ones included — the
# alert began when its first object appeared, exported or not.
alert_recorded_at = func.min(Sequence.recorded_at)
# The platform scores an alert, not an object: the verdict rides the
# primary lane and every object-split sibling stays NULL by import
# construction, so max() collapses the group without losing anything.
# Spans ALL lanes like alert_recorded_at — an unsure primary lane must
# not erase its alert's score.
temporal_model_score = func.max(Sequence.temporal_model_score)
temporal_model_version = func.max(Sequence.temporal_model_version)
temporal_api_version = func.max(Sequence.temporal_api_version)

stmt = (
select(
Sequence.source_api.label("source_api"),
Sequence.platform_alert_id.label("platform_alert_id"),
alert_recorded_at.label("recorded_at"),
last_annotated_at.label("last_annotated_at"),
temporal_model_score.label("temporal_model_score"),
temporal_model_version.label("temporal_model_version"),
temporal_api_version.label("temporal_api_version"),
)
.select_from(Sequence)
.outerjoin(SequenceAnnotation, SequenceAnnotation.sequence_id == Sequence.id)
Expand Down Expand Up @@ -394,6 +420,9 @@ async def export_alerts(
azimuth=first_seq.azimuth,
recorded_at=row.recorded_at,
last_annotated_at=row.last_annotated_at,
temporal_model_score=row.temporal_model_score,
temporal_model_version=row.temporal_model_version,
temporal_api_version=row.temporal_api_version,
objects=objects,
)
)
Expand Down
156 changes: 156 additions & 0 deletions annotation_api/src/tests/endpoints/test_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ async def create_lane(
organisation_name: str = "Export Org",
organisation_id: int = 70,
recorded_at: Optional[datetime] = None,
temporal_model_score: Optional[float] = None,
temporal_model_version: Optional[str] = None,
temporal_api_version: Optional[str] = None,
) -> int:
"""Create one sequence (lane) of an alert, returns sequence id."""
payload = {
Expand All @@ -73,6 +76,14 @@ async def create_lane(
"recorded_at": (recorded_at or now).isoformat(),
"last_seen_at": (recorded_at or now).isoformat(),
}
# Omitted rather than sent empty: the platform scores an alert, so only
# its primary lane carries these; siblings must stay NULL.
if temporal_model_score is not None:
payload["temporal_model_score"] = str(temporal_model_score)
if temporal_model_version is not None:
payload["temporal_model_version"] = temporal_model_version
if temporal_api_version is not None:
payload["temporal_api_version"] = temporal_api_version
resp = await client.post("/sequences", data=payload)
assert resp.status_code == 201, resp.text
return resp.json()["id"]
Expand Down Expand Up @@ -874,6 +885,151 @@ async def test_export_alerts_annotation_updated_watermark(
assert [i["platform_alert_id"] for i in items] == [7602]


@pytest.mark.asyncio
async def test_export_alerts_carries_alert_temporal_score(
authenticated_client: AsyncClient,
sequence_session,
detection_session,
dummy_bucket,
):
"""The platform's temporal-model verdict is an ALERT property: it rides
the scored primary lane, and the export surfaces it once at alert level
even though the sibling lane holds NULL."""
scored_seq = await create_lane(
authenticated_client,
platform_alert_id=7801,
alert_api_id=7801,
temporal_model_score=0.87,
temporal_model_version="0.2.0",
temporal_api_version="0.3.1",
)
sibling_seq = await create_lane(
authenticated_client, platform_alert_id=7801, alert_api_id=1000007801001
)
for seq_id in (scored_seq, sibling_seq):
det_id = await create_frame(
authenticated_client, sequence_id=seq_id, alert_api_id=1
)
await annotate_lane(
authenticated_client,
sequence_id=seq_id,
detection_ids=[det_id],
is_smoke=False,
false_positive_types=["antenna"],
)

resp = await authenticated_client.get("/export/alerts")
assert resp.status_code == 200, resp.text
items = resp.json()["items"]
assert len(items) == 1
alert = items[0]
assert alert["temporal_model_score"] == 0.87
assert alert["temporal_model_version"] == "0.2.0"
assert alert["temporal_api_version"] == "0.3.1"


@pytest.mark.asyncio
async def test_export_alerts_temporal_score_null_when_never_scored(
authenticated_client: AsyncClient,
sequence_session,
detection_session,
dummy_bucket,
):
"""An alert the platform never scored exports null, never 0.0 — the
difference is 'no verdict' vs 'confidently not smoke'."""
await seed_minimal_fp_alert(authenticated_client, platform_alert_id=7802)

resp = await authenticated_client.get("/export/alerts")
alert = resp.json()["items"][0]
assert alert["temporal_model_score"] is None
assert alert["temporal_model_version"] is None
assert alert["temporal_api_version"] is None


@pytest.mark.asyncio
async def test_export_alerts_temporal_score_zero_survives(
authenticated_client: AsyncClient,
sequence_session,
detection_session,
dummy_bucket,
):
"""0.0 is a real production verdict and must not be flattened to null by
any falsy-value handling on the way out."""
seq_id = await create_lane(
authenticated_client,
platform_alert_id=7803,
alert_api_id=7803,
temporal_model_score=0.0,
temporal_model_version="0.2.0",
)
det_id = await create_frame(
authenticated_client, sequence_id=seq_id, alert_api_id=1
)
await annotate_lane(
authenticated_client,
sequence_id=seq_id,
detection_ids=[det_id],
is_smoke=False,
false_positive_types=["antenna"],
)

resp = await authenticated_client.get("/export/alerts")
alert = resp.json()["items"][0]
assert alert["temporal_model_score"] == 0.0
assert alert["temporal_model_score"] is not None


@pytest.mark.asyncio
async def test_export_alerts_temporal_score_survives_unsure_scored_lane(
authenticated_client: AsyncClient,
sequence_session,
detection_session,
dummy_bucket,
):
"""The score aggregates over ALL lanes, not just exported ones: when the
scored lane is unsure (and so omitted from objects), the alert it belongs
to must still report the platform's verdict."""
unsure_scored_seq = await create_lane(
authenticated_client,
platform_alert_id=7804,
alert_api_id=7804,
temporal_model_score=0.42,
temporal_model_version="0.2.0",
)
sure_seq = await create_lane(
authenticated_client, platform_alert_id=7804, alert_api_id=1000007804001
)
unsure_det = await create_frame(
authenticated_client, sequence_id=unsure_scored_seq, alert_api_id=1
)
sure_det = await create_frame(
authenticated_client, sequence_id=sure_seq, alert_api_id=1
)
await annotate_lane(
authenticated_client,
sequence_id=unsure_scored_seq,
detection_ids=[unsure_det],
is_smoke=True,
smoke_type="wildfire",
is_unsure=True,
)
await annotate_lane(
authenticated_client,
sequence_id=sure_seq,
detection_ids=[sure_det],
is_smoke=False,
false_positive_types=["antenna"],
)

resp = await authenticated_client.get("/export/alerts")
items = resp.json()["items"]
assert len(items) == 1
alert = items[0]
assert [o["sequence_id"] for o in alert["objects"]] == [sure_seq]
assert alert["temporal_model_score"] == 0.42
assert alert["temporal_model_version"] == "0.2.0"


@pytest.mark.asyncio
async def test_export_alerts_requires_auth(async_client: AsyncClient):
resp = await async_client.get("/export/alerts")
Expand Down
15 changes: 15 additions & 0 deletions docs/specs/2026-08-06-export-alerts-endpoint-design.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ overlay row) restores the alert to the export untouched.
"azimuth": 200,
"recorded_at": "2026-07-14T15:42:10",
"last_annotated_at": "2026-07-16T10:03:17",
"temporal_model_score": 0.87,
"temporal_model_version": "0.2.0",
"temporal_api_version": "0.3.1",
"objects": [
{
"sequence_id": 3121,
Expand Down Expand Up @@ -136,6 +139,18 @@ azimuth, `recorded_at` (alert start, min over lanes' `recorded_at`), and
annotations) across all exported lanes. `last_annotated_at` is the value the
`annotation_updated_gte` filter compares against.

Also alert level: `temporal_model_score` and its provenance pair
`temporal_model_version` / `temporal_api_version`, each a `max` over the
alert's lanes. Alert grain is the score's true grain — the platform scores a
platform sequence, and object-splitting is annotator-side, so the verdict
rides the primary lane while split siblings stay NULL and `max` collapses the
group losslessly. Unlike `last_annotated_at`, these span **all** lanes rather
than only exported ones, so an unsure primary lane cannot erase its alert's
score. `null` means no verdict is attributed (never scored, or scored but not
attributable to a lane during the split) — never "scored low"; `0.0` is a real
verdict. A temporal-score refresh does not move `last_annotated_at`, so
backfilled scores require a full pull rather than an incremental one.

**Object level:**

- `sequence_id` is the lane/track identity. The importer's synthetic `alert_api_id`
Expand Down
Loading