feat: Allow nameless scorer results

py/src/braintrust/framework.py

@@ -51,7 +51,17 @@
     validate_parameters,
 )
 from .resource_manager import ResourceManager
-from .score import Classification, ClassificationItem, Score, ScoreLike, is_classification, is_score, is_scorer
+from .score import (
+    Classification,
+    ClassificationItem,
+    NamedScoreDict,
+    Score,
+    ScoreDict,
+    ScoreLike,
+    is_classification,
+    is_score,
+    is_scorer,
+)
 from .serializable_data_class import SerializableDataClass
 from .span_types import SpanTypeAttribute
 from .types._eval import EvalCaseDict, EvalCaseDictNoOutput, ExperimentDatasetEvent
@@ -224,7 +234,9 @@ class EvalScorerArgs(SerializableDataClass, Generic[Input, Output, Expected]):
     tags: Sequence[str] | None = None
 
 
-OneOrMoreScores = float | int | bool | None | ScoreLike | Sequence[ScoreLike]
+OneOrMoreScores = (
+    float | int | bool | None | ScoreLike | ScoreDict | NamedScoreDict | Sequence[ScoreLike | NamedScoreDict]
+)
 OneOrMoreClassifications = None | Classification | Mapping[str, Any] | list[Classification | Mapping[str, Any]]
 
 
@@ -336,7 +348,9 @@ class Evaluator(Generic[Input, Output, Expected]):
     scores: Sequence[EvalScorer[Input, Output, Expected]]
     """
     A list of scorers to evaluate the results of the task. Each scorer can be a Scorer object or a function
-    that takes `input`, `output`, and `expected` arguments and returns a `Score` object. The function can be async.
+    that takes `input`, `output`, and `expected` arguments. It can return a number, a `Score` object,
+    a `ScoreDict`, or a sequence of named scores. A single dict can omit `name` to use the scorer's name;
+    each score in a sequence must include a name. The function can be async.
     """
 
     experiment_name: str | None
@@ -854,7 +868,8 @@ async def EvalAsync(
     :param data: Returns an iterator over the evaluation dataset. Each element of the iterator should be a `EvalCase`.
     :param task: Runs the evaluation task on a single input. The `hooks` object can be used to add metadata to the evaluation.
     :param scores: A list of scorers to evaluate the results of the task. Each scorer can be a Scorer object or a function
-    that takes an `EvalScorerArgs` object and returns a `Score` object.
+    that returns a number, a `Score` object, a `ScoreDict`, or a sequence of named scores.
+    A single dict can omit `name` to use the scorer's name; each score in a sequence must include a name.
     :param experiment_name: (Optional) Experiment name. If not specified, a name will be generated automatically.
     :param trial_count: The number of times to run the evaluator per input. This is useful for evaluating applications that
     have non-deterministic behavior and gives you both a stronger aggregate measure and a sense of the variance in the results.
@@ -982,7 +997,8 @@ def Eval(
     :param data: Returns an iterator over the evaluation dataset. Each element of the iterator should be a `EvalCase`.
     :param task: Runs the evaluation task on a single input. The `hooks` object can be used to add metadata to the evaluation.
     :param scores: A list of scorers to evaluate the results of the task. Each scorer can be a Scorer object or a function
-    that takes an `EvalScorerArgs` object and returns a `Score` object.
+    that returns a number, a `Score` object, a `ScoreDict`, or a sequence of named scores.
+    A single dict can omit `name` to use the scorer's name; each score in a sequence must include a name.
     :param experiment_name: (Optional) Experiment name. If not specified, a name will be generated automatically.
     :param trial_count: The number of times to run the evaluator per input. This is useful for evaluating applications that
     have non-deterministic behavior and gives you both a stronger aggregate measure and a sense of the variance in the results.
@@ -1510,6 +1526,8 @@ async def await_or_run_scorer(root_span, scorer, name, **kwargs):
 
             result = await call_user_fn(event_loop, score, **kwargs)
             if isinstance(result, dict):
+                if "name" not in result and "score" in result:
+                    result = {"name": name, **result}
                 result = _normalize_score(result, "When returning a dict, it must be a valid Score object.")
 
             if isinstance(result, Iterable) and not isinstance(result, (str, bytes, Mapping)):
@@ -1965,6 +1983,8 @@ def build_local_summary(
     "EvalCase",
     "EvalHooks",
     "Evaluator",
+    "NamedScoreDict",
     "Reporter",
     "Score",
+    "ScoreDict",
 ]

py/src/braintrust/score.py

@@ -54,6 +54,23 @@ def __post_init__(self):
             )
 
 
+class _ScoreDictFields(TypedDict):
+    score: float | None
+    metadata: NotRequired[Metadata]
+
+
+class ScoreDict(_ScoreDictFields):
+    """A single score result. When omitted, name defaults to the scorer's name."""
+
+    name: NotRequired[str]
+
+
+class NamedScoreDict(_ScoreDictFields):
+    """A score result with an explicit name, required when returning multiple scores."""
+
+    name: str
+
+
 class ScoreLike(Protocol):
     @property
     def name(self) -> str: ...

py/src/braintrust/test_framework.py

@@ -441,6 +441,109 @@ def _run_eval_sync(self, *args, **kwargs):
         assert result.summary.scores[scorer_name].score == 1.0
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("is_async", [False, True])
+@pytest.mark.parametrize(
+    "score_result",
+    [
+        {"score": 1.0},
+        {"score": None},
+        {"score": 0.0, "metadata": {"reason": "No match"}},
+        {"name": "custom", "score": 0.5, "metadata": {"reason": "Partial match"}},
+        {"name": "", "score": 1.0},
+        {"name": "skipped"},
+    ],
+)
+async def test_run_evaluator_normalizes_single_dict_score(
+    score_result, is_async, with_memory_logger, with_simulate_login
+):
+    original_result = score_result.copy()
+
+    def scorer(input_value, output, expected):
+        return score_result
+
+    async def async_scorer(input_value, output, expected):
+        return score_result
+
+    scorer_fn = async_scorer if is_async else scorer
+    expected_scores = {score_result.get("name", scorer_fn.__name__): score_result.get("score")}
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-single-dict-score",
+        data=[EvalCase(input=1, expected=1)],
+        task=lambda input_value: input_value,
+        scores=[scorer_fn],
+        experiment_name=None,
+        metadata=None,
+        summarize_scores=False,
+    )
+    exp = init_test_exp("test-single-dict-score", "test-project")
+    result = await run_evaluator(exp, evaluator, None, [])
+
+    assert result.results[0].scores == expected_scores
+    assert "scorer_errors" not in result.results[0].metadata
+    assert score_result == original_result
+    score_spans = [log for log in with_memory_logger.pop() if log.get("span_attributes", {}).get("type") == "score"]
+    assert len(score_spans) == 1
+    assert score_spans[0]["scores"] == expected_scores
+    assert score_spans[0]["output"] == {"score": score_result.get("score")}
+    assert score_spans[0].get("metadata", {}) == score_result.get("metadata", {})
+
+
+@pytest.mark.asyncio
+async def test_run_evaluator_preserves_named_dict_subclass():
+    class PercentageScore(dict):
+        def items(self):
+            for key, value in super().items():
+                yield key, value / 100 if key == "score" else value
+
+    def scorer(input_value, output, expected):
+        return PercentageScore(name="percentage", score=80)
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-dict-subclass-score",
+        data=[EvalCase(input=1, expected=1)],
+        task=lambda input_value: input_value,
+        scores=[scorer],
+        experiment_name=None,
+        metadata=None,
+    )
+    result = await run_evaluator(None, evaluator, None, [])
+
+    assert result.results[0].scores == {"percentage": 0.8}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "score_result",
+    [
+        [{"score": 1.0}],
+        [{"name": "named", "score": 1.0}, {"score": 0.5}],
+        {},
+        {"metadata": {"reason": "Missing score"}},
+        {"score": 2.0},
+    ],
+)
+async def test_run_evaluator_rejects_invalid_dict_scores(score_result):
+    def scorer(input_value, output, expected):
+        return score_result
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-invalid-dict-scores",
+        data=[EvalCase(input=1, expected=1)],
+        task=lambda input_value: input_value,
+        scores=[scorer],
+        experiment_name=None,
+        metadata=None,
+    )
+    result = await run_evaluator(None, evaluator, None, [])
+
+    assert result.results[0].scores == {}
+    assert "valid Score object" in result.results[0].metadata["scorer_errors"]["scorer"]
+
+
 @pytest.mark.asyncio
 async def test_run_evaluator_normalizes_list_of_dict_scores():
     data = [

py/src/braintrust/type_tests/test_dict_scorers.py

@@ -0,0 +1,100 @@
+"""Type-check and runtime coverage for dictionary scorer returns."""
+
+from collections.abc import Sequence
+
+import pytest
+from braintrust import NamedScoreDict, ScoreDict
+from braintrust.framework import Eval, EvalAsync, EvalCase, EvalScorer, OneOrMoreScores
+from braintrust.score import Score, ScoreLike
+
+
+LegacyScores = float | int | bool | None | ScoreLike | Sequence[ScoreLike]
+
+
+@pytest.mark.parametrize(
+    "value,expected_scores",
+    [
+        (1.0, {"scorer": 1.0}),
+        (0, {"scorer": 0}),
+        (True, {"scorer": True}),
+        (False, {"scorer": False}),
+        (None, {"scorer": None}),
+        (Score(name="named", score=0.5), {"named": 0.5}),
+        ([Score(name="named", score=0.5)], {"named": 0.5}),
+        ((Score(name="named", score=0.5),), {"named": 0.5}),
+    ],
+)
+def test_eval_accepts_legacy_return_annotation(value: LegacyScores, expected_scores: dict[str, float | None]) -> None:
+    def scorer(input: str, output: str, expected: str | None = None) -> LegacyScores:
+        return value
+
+    typed_scorer: EvalScorer[str, str, str] = scorer
+    result = Eval(
+        "test-legacy-scorer",
+        data=[EvalCase(input="hello", expected="hello")],
+        task=lambda input: input,
+        scores=[typed_scorer],
+        no_send_logs=True,
+    )
+
+    assert result.results[0].scores == expected_scores
+
+
+def test_eval_accepts_single_dict_scorer() -> None:
+    def scorer(input: str, output: str, expected: str | None = None) -> ScoreDict:
+        return {"score": 1.0, "metadata": {"reason": "Matches"}}
+
+    typed_scorer: EvalScorer[str, str, str] = scorer
+    result = Eval(
+        "test-dict-scorers",
+        data=[EvalCase(input="hello", expected="hello")],
+        task=lambda input: input,
+        scores=[typed_scorer],
+        no_send_logs=True,
+    )
+
+    assert result.results[0].scores == {"scorer": 1.0}
+
+
+@pytest.mark.asyncio
+async def test_eval_async_accepts_single_dict_scorer() -> None:
+    async def scorer(input: str, output: str, expected: str | None = None) -> OneOrMoreScores:
+        return {"score": None}
+
+    typed_scorer: EvalScorer[str, str, str] = scorer
+    result = await EvalAsync(
+        "test-dict-scorers",
+        data=[EvalCase(input="hello", expected="hello")],
+        task=lambda input: input,
+        scores=[typed_scorer],
+        no_send_logs=True,
+    )
+
+    assert result.results[0].scores == {"scorer": None}
+
+
+def test_eval_accepts_named_dict_scores() -> None:
+    def scorer(input: str, output: str, expected: str | None = None) -> list[NamedScoreDict]:
+        return [{"name": "match", "score": 1.0}, {"name": "quality", "score": 0.5}]
+
+    result = Eval(
+        "test-dict-scorers",
+        data=[EvalCase(input="hello", expected="hello")],
+        task=lambda input: input,
+        scores=[scorer],
+        no_send_logs=True,
+    )
+
+    assert result.results[0].scores == {"match": 1.0, "quality": 0.5}
+
+
+def test_eval_accepts_inline_dict_scorer() -> None:
+    result = Eval(
+        "test-dict-scorers",
+        data=[EvalCase(input="hello", expected="hello")],
+        task=lambda input: input,
+        scores=[lambda input, output, expected: ScoreDict(score=1.0)],
+        no_send_logs=True,
+    )
+
+    assert result.results[0].scores == {"scorer_0": 1.0}