Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions apps/backend/services/metrics_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
from models.scan import SCAN_STATUS_VALUES, VULN_SEVERITY_VALUES
from models.task_run import TaskRun
from services.policy_gate import _CLOSED_FINDING_STATUSES
from services.trivy_health_service import get_trivy_db_status_cached

log = structlog.get_logger("services.metrics")

Expand Down Expand Up @@ -177,6 +178,36 @@ async def _task_run_last_recorded(session: AsyncSession) -> float:
return newest.timestamp() if newest else 0.0



def _vuln_db_freshness() -> tuple[float, float]:
"""When the vulnerability database was last updated, and our refresh gap.

Two values because neither answers the question alone. The timestamp says
when the data stopped changing; the interval says how long that is
supposed to be able to go on. A collector that has both can decide
staleness for the deployment it is watching, which is the only place that
decision can be made correctly: schedules differ, and an air-gapped
install that mirrors the database on a slower cadence is not broken.

No threshold is computed here, and the panel's own fresh / stale
classification is deliberately not published. That bucketing exists for a
screen somebody is looking at; a series carrying it would freeze one
judgement into the metric and make the collector's own rule unreachable.

Reads the cached snapshot. The uncached read touches the filesystem, and
this runs on every scrape.

Returns ``(0.0, interval)`` when no database has been downloaded yet.
Zero is distinguishable from any real timestamp and keeps the series
present, which matters: a missing series draws nothing on a dashboard and
reads as nothing being wrong, and "no database at all" is the worst state
this can be in.
"""
status = get_trivy_db_status_cached()
last_update = status.last_update.timestamp() if status.last_update else 0.0
return last_update, float(status.refresh_interval_hours)


async def render_metrics(session: AsyncSession) -> str:
"""The whole document, in the order the contract file lists it.

Expand Down Expand Up @@ -320,6 +351,29 @@ async def render_metrics(session: AsyncSession) -> str:
)
)

vuln_db_last_update, vuln_db_interval_hours = await asyncio.to_thread(
_vuln_db_freshness
)
document.append(
_block(
"trusca_vuln_db_last_update_timestamp_seconds",
"Unix time the vulnerability database was last updated upstream; "
"0 when none has been downloaded. Scans keep succeeding against a "
"stale database, so nothing else reports this.",
"gauge",
[({}, vuln_db_last_update)],
)
)
document.append(
_block(
"trusca_vuln_db_refresh_interval_hours",
"Configured hours between refresh attempts. Published so a "
"collector can derive staleness from this deployment's own "
"cadence rather than a threshold baked in here.",
"gauge",
[({}, vuln_db_interval_hours)],
)
)
document.append(
_block(
"trusca_task_runs_last_recorded_timestamp_seconds",
Expand Down
131 changes: 131 additions & 0 deletions apps/backend/tests/unit/services/test_vuln_db_freshness_metric.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2026 TRUSCA contributors
"""The vulnerability-database freshness series, and what it refuses to decide.

A database that stopped updating does not make scans fail. They keep running,
keep finding the vulnerabilities it knew about when it stopped, and report
success. Nothing in the system objects. That is the whole reason for a series
here rather than a check somewhere.

Two things are asserted beyond the happy path. The series stays present when
no database exists at all, because a missing series draws nothing and reads as
nothing being wrong. And no staleness verdict is published, because the
deployment's own refresh cadence is the only correct basis for one and that
lives in the collector.
"""

from __future__ import annotations

from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any

import pytest

from services import metrics_service


@dataclass
class _Status:
"""Only the two fields the metric reads."""

last_update: datetime | None
refresh_interval_hours: int


@pytest.fixture
def status(monkeypatch: pytest.MonkeyPatch) -> Any:
def _install(snapshot: _Status) -> None:
monkeypatch.setattr(
metrics_service, "get_trivy_db_status_cached", lambda: snapshot
)

return _install


def test_the_last_update_is_published_as_unix_time(status: Any) -> None:
moment = datetime(2026, 7, 18, 12, 57, 54, tzinfo=UTC)
status(_Status(last_update=moment, refresh_interval_hours=168))

last_update, _ = metrics_service._vuln_db_freshness()

assert last_update == moment.timestamp()


def test_the_configured_interval_is_published_beside_it(status: Any) -> None:
"""The collector needs both to decide anything.

The timestamp says when the data stopped changing; the interval says how
long that is supposed to be able to go on. An air-gapped install mirroring
on a slower cadence is not broken, and only the interval distinguishes it
from one that is.
"""
status(_Status(last_update=datetime.now(UTC), refresh_interval_hours=168))

_, interval = metrics_service._vuln_db_freshness()

assert interval == 168.0


def test_no_database_still_publishes_the_series(status: Any) -> None:
"""Zero, not an absent series.

"No database has ever been downloaded" is the worst state this can be in,
and it is exactly the state where a series that disappeared would leave a
dashboard looking clean.
"""
status(_Status(last_update=None, refresh_interval_hours=168))

last_update, interval = metrics_service._vuln_db_freshness()

assert last_update == 0.0
assert interval == 168.0


def test_a_long_stale_database_is_reported_as_a_plain_timestamp(
status: Any,
) -> None:
"""No verdict, no clamping, no special value for "too old".

A real deployment was found 46 days behind while every scan succeeded. The
series reports the timestamp it has; whether 46 days is a problem depends
on the refresh cadence, which is why that is published alongside rather
than folded into a judgement here.
"""
stale = datetime(2026, 7, 18, 12, 57, 54, tzinfo=UTC)
status(_Status(last_update=stale, refresh_interval_hours=168))

last_update, _ = metrics_service._vuln_db_freshness()

assert last_update == stale.timestamp()


def test_the_accessor_never_reads_the_panel_s_verdict() -> None:
"""The fresh / stale bucket the snapshot carries is not published.

That classification exists for a screen. A series carrying it would freeze
one judgement into the metric and put it beyond the collector's reach: a
deployment cannot override a label it is handed, and the right threshold
depends on a refresh cadence only the deployment knows.

Asserted on the attribute access rather than on words in the source. The
first two versions of this test matched the strings "freshness" and
"fresh", and both failed on names that legitimately contain them: the
accessor is named for what it measures, and "refresh" contains "fresh".
A test of spelling is not a test of behaviour.
"""
import ast
import inspect

tree = ast.parse(inspect.getsource(metrics_service._vuln_db_freshness))
read = {
node.attr
for node in ast.walk(tree)
if isinstance(node, ast.Attribute)
}

assert "freshness" not in read, (
"the metric reads the snapshot's freshness bucket; publish the "
"timestamp and the interval and let the collector decide"
)
assert read >= {"last_update", "refresh_interval_hours"}
16 changes: 16 additions & 0 deletions docs-site/docs/admin-guide/disk-and-health.md
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,8 @@ What it publishes is a fixed list of aggregate counts:
| `trusca_task_runs_24h` | `task`, `outcome` | Background task runs in the last day. `outcome=running` counts runs that started and never reported an end |
| `trusca_task_run_duration_seconds_p50_24h` | `task` | Median duration of runs that finished in the last day |
| `trusca_task_run_duration_seconds_p95_24h` | `task` | Same at the 95th percentile |
| `trusca_vuln_db_last_update_timestamp_seconds` | | Unix time the vulnerability database was last updated upstream, `0` when none has been downloaded |
| `trusca_vuln_db_refresh_interval_hours` | | Configured hours between refresh attempts |
| `trusca_task_runs_last_recorded_timestamp_seconds` | | Unix time of the newest task-run row, `0` when the table is empty |

No project, package, repository or person's name appears anywhere in the
Expand All @@ -215,6 +217,20 @@ Task history is swept on a retention schedule, so a cumulative count would go
down when the sweep runs and a collector reads a falling counter as a restart.
The window is in the name because changing it changes what the series means.

The two vulnerability-database series belong together: the timestamp says when
the data stopped changing, the interval says how long that is supposed to be
able to go on. Neither answers on its own, because an air-gapped install
mirroring on a slower cadence is not broken and only the interval separates it
from one that is. No staleness verdict ships in the metrics, and the fresh /
stale wording the panel uses is deliberately not published: that bucketing is
for a screen somebody is reading, and a series carrying it would put the
judgement beyond your collector's reach.

This one deserves an alert because a stale database is silent. Scans keep
running, keep finding what the database knew when it stopped, and report
success. One deployment was found 46 days behind with every scan green, and
what surfaced it was somebody looking at the disk for an unrelated reason.

`trusca_task_runs_last_recorded_timestamp_seconds` is worth an alert of its
own. It watches the recorder rather than the work: recording history is
designed never to fail a task, so a missing database grant or an unrun
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,8 @@ docker-compose -f docker-compose.yml exec backend \
| `trusca_task_runs_24h` | `task`, `outcome` | 최근 하루 동안의 배경 작업 실행 수. `outcome=running`은 시작한 뒤 종료를 보고하지 않은 실행입니다 |
| `trusca_task_run_duration_seconds_p50_24h` | `task` | 최근 하루 동안 끝난 실행의 소요 시간 중앙값 |
| `trusca_task_run_duration_seconds_p95_24h` | `task` | 같은 값의 95 백분위수 |
| `trusca_vuln_db_last_update_timestamp_seconds` | | 취약점 데이터베이스가 업스트림에서 마지막으로 갱신된 유닉스 시각. 내려받은 적이 없으면 `0` |
| `trusca_vuln_db_refresh_interval_hours` | | 설정된 갱신 시도 간격(시간) |
| `trusca_task_runs_last_recorded_timestamp_seconds` | | 가장 최근 작업 이력 행의 유닉스 시각. 테이블이 비어 있으면 `0` |

프로젝트·패키지·저장소·사람 이름은 출력 어디에도 나오지 않습니다. 목록은
Expand All @@ -210,6 +212,16 @@ docker-compose -f docker-compose.yml exec backend \
카운터를 프로세스 재시작으로 읽습니다. 창 길이를 이름에 넣은 이유는 창이 바뀌면 계열의
뜻이 달라지기 때문입니다.

취약점 데이터베이스 계열 둘은 함께 봐야 합니다. 시각은 데이터가 언제 멈췄는지를, 간격은 그
상태가 얼마나 지속돼도 되는지를 말합니다. 하나만으로는 판단할 수 없습니다. 폐쇄망에서 더 성긴
주기로 미러링하는 배포는 고장이 아니고, 그것과 실제로 멈춘 배포를 가르는 것이 간격이기
때문입니다. 지표에는 신선도 판정을 넣지 않습니다. 관리자 화면이 쓰는 fresh·stale 구분은
사람이 보는 화면을 위한 것이고, 계열에 담으면 그 판단이 수집기 손을 벗어납니다.

이 값에 경보가 필요한 이유는 낡은 데이터베이스가 조용하기 때문입니다. 스캔은 계속 돌고,
데이터베이스가 멈춘 시점에 알던 것을 계속 찾아내고, 성공을 보고합니다. 어느 배포는 46일
뒤처진 채로 모든 스캔이 정상이었고, 그것을 드러낸 것은 다른 일로 디스크를 살펴보던 사람이었습니다.

`trusca_task_runs_last_recorded_timestamp_seconds`에는 별도 경보를 걸어 둘 만합니다.
이 값은 작업이 아니라 기록 장치를 봅니다. 이력 기록은 작업을 실패시키지 않도록 설계돼
있어서 데이터베이스 권한이 빠지거나 마이그레이션이 돌지 않아도 어디에도 오류가 나지
Expand Down
12 changes: 12 additions & 0 deletions tests/contracts/metrics-series.json
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,18 @@
],
"help": "Task duration in seconds, p95 over runs that finished in the last 24h."
},
{
"name": "trusca_vuln_db_last_update_timestamp_seconds",
"type": "gauge",
"labels": [],
"help": "Unix time the vulnerability database was last updated upstream; 0 when none has been downloaded. Scans keep succeeding against a stale database, so nothing else reports this."
},
{
"name": "trusca_vuln_db_refresh_interval_hours",
"type": "gauge",
"labels": [],
"help": "Configured hours between refresh attempts. Published so a collector can derive staleness from this deployment's own cadence rather than a threshold baked in here."
},
{
"name": "trusca_task_runs_last_recorded_timestamp_seconds",
"type": "gauge",
Expand Down
Loading