Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/usage/results-and-reporting.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,9 @@ from rampart.reporting import JsonFileReportSink
sink = JsonFileReportSink(output_dir=Path(".report"))
```

Output: `.report/run_report_2026-04-25T14-30-00.json`
Output: `.report/run_report_2026-04-25T14-30-00-123.json`

The filename contains a UTC timestamp with millisecond precision. If another report already has the same timestamp, a random UUID is appended to the new filename. Files are created atomically and existing reports are never overwritten. Reports written within the same millisecond have no defined filename order relative to each other.

### Custom Sinks

Expand Down
31 changes: 26 additions & 5 deletions rampart/reporting/json_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ def rampart_sinks():
import json
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from uuid import uuid4

from rampart.common.text import safe_float, safe_str, safe_str_list

Expand All @@ -45,8 +46,10 @@ def rampart_sinks():
class JsonFileReportSink:
"""Writes the test run report to a JSON file.

Each run produces a timestamped file:
``<output_dir>/run_report_2026-03-19T21-30-00.json``
Each run normally produces a file named ``run_report_<timestamp>.json``.
The UTC timestamp includes milliseconds. If that filename already exists,
a UUID suffix distinguishes the colliding run. Existing files are never
overwritten.

Args:
output_dir (Path): Directory to write report files into.
Expand All @@ -62,14 +65,32 @@ async def emit_async(self, *, report: TestRunReport) -> None:

Args:
report (TestRunReport): The aggregated test run results.

Raises:
FileExistsError: If the random fallback filename also exists, or
``output_dir`` exists and is not a directory.
"""
self._output_dir.mkdir(parents=True, exist_ok=True)

timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S")
timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S-%f")[:-3]
data = self._serialize_report(report)
content = json.dumps(data, indent=2, default=str)

filepath = self._output_dir / f"run_report_{timestamp}.json"
try:
report_file = filepath.open("x", encoding="utf-8")
except FileExistsError:
filepath = self._output_dir / f"run_report_{timestamp}_{uuid4().hex}.json"

data = self._serialize_report(report)
filepath.write_text(json.dumps(data, indent=2, default=str))
# Leave the exception handler before opening the fallback so a
# second collision reports only the path that actually collided.
report_file = None

if report_file is None:
report_file = filepath.open("x", encoding="utf-8")

with report_file:
report_file.write(content)

def _serialize_report(self, report: TestRunReport) -> dict[str, Any]:
"""Convert a TestRunReport to a JSON-serializable dict.
Expand Down
90 changes: 90 additions & 0 deletions tests/unit/reporting/test_json_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,11 @@
from __future__ import annotations

import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from unittest.mock import patch
from uuid import UUID

import pytest

Expand Down Expand Up @@ -366,6 +369,93 @@ async def test_emitted_file_contains_metadata_async(self, tmp_path: Path) -> Non
"page_url": "https://example.com/chat",
}

async def test_same_timestamp_preserves_every_report_async(
self,
tmp_path: Path,
) -> None:
sink = JsonFileReportSink(output_dir=tmp_path)
fixed = datetime(2026, 8, 27, 12, 0, 0, 123456, tzinfo=UTC)

with patch("rampart.reporting.json_file.datetime") as clock:
clock.now.return_value = fixed
for run in range(3):
await sink.emit_async(report=TestRunReport(metadata={"run": run}))
clock.now.assert_called_with(UTC)

files = list(tmp_path.glob("run_report_*.json"))
assert len(files) == 3
assert {
json.loads(path.read_text(encoding="utf-8"))["metadata"]["run"]
for path in files
} == {0, 1, 2}

concise = tmp_path / "run_report_2026-08-27T12-00-00-123.json"
assert json.loads(concise.read_text(encoding="utf-8"))["metadata"] == {
"run": 0,
}
colliding_files = [path for path in files if path != concise]
assert len(colliding_files) == 2
for path in colliding_files:
assert path.name.startswith("run_report_2026-08-27T12-00-00-123_")
identifier = path.stem.rsplit("_", 1)[1]
assert len(identifier) == 32
assert UUID(hex=identifier).version == 4

async def test_existing_report_is_not_replaced_async(self, tmp_path: Path) -> None:
original = tmp_path / "run_report_2026-08-27T12-00-00-000.json"
original.write_text("keep me", encoding="utf-8")
sink = JsonFileReportSink(output_dir=tmp_path)
fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC)

with patch("rampart.reporting.json_file.datetime") as clock:
clock.now.return_value = fixed
await sink.emit_async(report=TestRunReport(metadata={"run": "new"}))

assert original.read_text(encoding="utf-8") == "keep me"
new_files = list(tmp_path.glob("run_report_2026-08-27T12-00-00-000_*.json"))
assert len(new_files) == 1
assert json.loads(new_files[0].read_text(encoding="utf-8"))["metadata"] == {
"run": "new",
}

async def test_uuid_collision_does_not_overwrite_existing_report_async(
self,
tmp_path: Path,
) -> None:
identifier = UUID("a3f18c92-654d-4b75-ad15-687d383d951b")
timestamp_file = tmp_path / "run_report_2026-08-27T12-00-00-000.json"
timestamp_file.write_text("keep timestamp", encoding="utf-8")
uuid_file = (
tmp_path / f"run_report_2026-08-27T12-00-00-000_{identifier.hex}.json"
)
uuid_file.write_text("keep uuid", encoding="utf-8")
sink = JsonFileReportSink(output_dir=tmp_path)
fixed = datetime(2026, 8, 27, 12, 0, 0, tzinfo=UTC)

with (
patch("rampart.reporting.json_file.datetime") as clock,
patch("rampart.reporting.json_file.uuid4", return_value=identifier),
):
clock.now.return_value = fixed
with pytest.raises(FileExistsError, match=identifier.hex):
await sink.emit_async(report=TestRunReport())

assert timestamp_file.read_text(encoding="utf-8") == "keep timestamp"
assert uuid_file.read_text(encoding="utf-8") == "keep uuid"
assert set(tmp_path.glob("run_report_*.json")) == {timestamp_file, uuid_file}

async def test_serialization_failure_does_not_create_a_file_async(
self,
tmp_path: Path,
) -> None:
sink = JsonFileReportSink(output_dir=tmp_path)
report = TestRunReport(metadata={"bad": {("tuple", "key"): "value"}})

with pytest.raises(TypeError, match="keys must be"):
await sink.emit_async(report=report)

assert list(tmp_path.glob("run_report_*.json")) == []


class TestReportMetadata:
"""Run-level TestRunReport.metadata is projected into the JSON output."""
Expand Down