diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py index e848d365..26780076 100644 --- a/py/src/braintrust/framework.py +++ b/py/src/braintrust/framework.py @@ -51,7 +51,17 @@ validate_parameters, ) from .resource_manager import ResourceManager -from .score import Classification, ClassificationItem, Score, ScoreLike, is_classification, is_score, is_scorer +from .score import ( + Classification, + ClassificationItem, + NamedScoreDict, + Score, + ScoreDict, + ScoreLike, + is_classification, + is_score, + is_scorer, +) from .serializable_data_class import SerializableDataClass from .span_types import SpanTypeAttribute from .types._eval import EvalCaseDict, EvalCaseDictNoOutput, ExperimentDatasetEvent @@ -224,7 +234,9 @@ class EvalScorerArgs(SerializableDataClass, Generic[Input, Output, Expected]): tags: Sequence[str] | None = None -OneOrMoreScores = float | int | bool | None | ScoreLike | Sequence[ScoreLike] +OneOrMoreScores = ( + float | int | bool | None | ScoreLike | ScoreDict | NamedScoreDict | Sequence[ScoreLike | NamedScoreDict] +) OneOrMoreClassifications = None | Classification | Mapping[str, Any] | list[Classification | Mapping[str, Any]] @@ -336,7 +348,9 @@ class Evaluator(Generic[Input, Output, Expected]): scores: Sequence[EvalScorer[Input, Output, Expected]] """ A list of scorers to evaluate the results of the task. Each scorer can be a Scorer object or a function - that takes `input`, `output`, and `expected` arguments and returns a `Score` object. The function can be async. + that takes `input`, `output`, and `expected` arguments. It can return a number, a `Score` object, + a `ScoreDict`, or a sequence of named scores. A single dict can omit `name` to use the scorer's name; + each score in a sequence must include a name. The function can be async. """ experiment_name: str | None @@ -854,7 +868,8 @@ async def EvalAsync( :param data: Returns an iterator over the evaluation dataset. Each element of the iterator should be a `EvalCase`. :param task: Runs the evaluation task on a single input. The `hooks` object can be used to add metadata to the evaluation. :param scores: A list of scorers to evaluate the results of the task. Each scorer can be a Scorer object or a function - that takes an `EvalScorerArgs` object and returns a `Score` object. + that returns a number, a `Score` object, a `ScoreDict`, or a sequence of named scores. + A single dict can omit `name` to use the scorer's name; each score in a sequence must include a name. :param experiment_name: (Optional) Experiment name. If not specified, a name will be generated automatically. :param trial_count: The number of times to run the evaluator per input. This is useful for evaluating applications that have non-deterministic behavior and gives you both a stronger aggregate measure and a sense of the variance in the results. @@ -982,7 +997,8 @@ def Eval( :param data: Returns an iterator over the evaluation dataset. Each element of the iterator should be a `EvalCase`. :param task: Runs the evaluation task on a single input. The `hooks` object can be used to add metadata to the evaluation. :param scores: A list of scorers to evaluate the results of the task. Each scorer can be a Scorer object or a function - that takes an `EvalScorerArgs` object and returns a `Score` object. + that returns a number, a `Score` object, a `ScoreDict`, or a sequence of named scores. + A single dict can omit `name` to use the scorer's name; each score in a sequence must include a name. :param experiment_name: (Optional) Experiment name. If not specified, a name will be generated automatically. :param trial_count: The number of times to run the evaluator per input. This is useful for evaluating applications that have non-deterministic behavior and gives you both a stronger aggregate measure and a sense of the variance in the results. @@ -1510,6 +1526,8 @@ async def await_or_run_scorer(root_span, scorer, name, **kwargs): result = await call_user_fn(event_loop, score, **kwargs) if isinstance(result, dict): + if "name" not in result and "score" in result: + result = {"name": name, **result} result = _normalize_score(result, "When returning a dict, it must be a valid Score object.") if isinstance(result, Iterable) and not isinstance(result, (str, bytes, Mapping)): @@ -1965,6 +1983,8 @@ def build_local_summary( "EvalCase", "EvalHooks", "Evaluator", + "NamedScoreDict", "Reporter", "Score", + "ScoreDict", ] diff --git a/py/src/braintrust/score.py b/py/src/braintrust/score.py index 43515669..c9395c49 100644 --- a/py/src/braintrust/score.py +++ b/py/src/braintrust/score.py @@ -54,6 +54,23 @@ def __post_init__(self): ) +class _ScoreDictFields(TypedDict): + score: float | None + metadata: NotRequired[Metadata] + + +class ScoreDict(_ScoreDictFields): + """A single score result. When omitted, name defaults to the scorer's name.""" + + name: NotRequired[str] + + +class NamedScoreDict(_ScoreDictFields): + """A score result with an explicit name, required when returning multiple scores.""" + + name: str + + class ScoreLike(Protocol): @property def name(self) -> str: ... diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py index 74dfb510..9a59c987 100644 --- a/py/src/braintrust/test_framework.py +++ b/py/src/braintrust/test_framework.py @@ -441,6 +441,109 @@ def _run_eval_sync(self, *args, **kwargs): assert result.summary.scores[scorer_name].score == 1.0 +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize( + "score_result", + [ + {"score": 1.0}, + {"score": None}, + {"score": 0.0, "metadata": {"reason": "No match"}}, + {"name": "custom", "score": 0.5, "metadata": {"reason": "Partial match"}}, + {"name": "", "score": 1.0}, + {"name": "skipped"}, + ], +) +async def test_run_evaluator_normalizes_single_dict_score( + score_result, is_async, with_memory_logger, with_simulate_login +): + original_result = score_result.copy() + + def scorer(input_value, output, expected): + return score_result + + async def async_scorer(input_value, output, expected): + return score_result + + scorer_fn = async_scorer if is_async else scorer + expected_scores = {score_result.get("name", scorer_fn.__name__): score_result.get("score")} + evaluator = Evaluator( + project_name="test-project", + eval_name="test-single-dict-score", + data=[EvalCase(input=1, expected=1)], + task=lambda input_value: input_value, + scores=[scorer_fn], + experiment_name=None, + metadata=None, + summarize_scores=False, + ) + exp = init_test_exp("test-single-dict-score", "test-project") + result = await run_evaluator(exp, evaluator, None, []) + + assert result.results[0].scores == expected_scores + assert "scorer_errors" not in result.results[0].metadata + assert score_result == original_result + score_spans = [log for log in with_memory_logger.pop() if log.get("span_attributes", {}).get("type") == "score"] + assert len(score_spans) == 1 + assert score_spans[0]["scores"] == expected_scores + assert score_spans[0]["output"] == {"score": score_result.get("score")} + assert score_spans[0].get("metadata", {}) == score_result.get("metadata", {}) + + +@pytest.mark.asyncio +async def test_run_evaluator_preserves_named_dict_subclass(): + class PercentageScore(dict): + def items(self): + for key, value in super().items(): + yield key, value / 100 if key == "score" else value + + def scorer(input_value, output, expected): + return PercentageScore(name="percentage", score=80) + + evaluator = Evaluator( + project_name="test-project", + eval_name="test-dict-subclass-score", + data=[EvalCase(input=1, expected=1)], + task=lambda input_value: input_value, + scores=[scorer], + experiment_name=None, + metadata=None, + ) + result = await run_evaluator(None, evaluator, None, []) + + assert result.results[0].scores == {"percentage": 0.8} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "score_result", + [ + [{"score": 1.0}], + [{"name": "named", "score": 1.0}, {"score": 0.5}], + {}, + {"metadata": {"reason": "Missing score"}}, + {"score": 2.0}, + ], +) +async def test_run_evaluator_rejects_invalid_dict_scores(score_result): + def scorer(input_value, output, expected): + return score_result + + evaluator = Evaluator( + project_name="test-project", + eval_name="test-invalid-dict-scores", + data=[EvalCase(input=1, expected=1)], + task=lambda input_value: input_value, + scores=[scorer], + experiment_name=None, + metadata=None, + ) + result = await run_evaluator(None, evaluator, None, []) + + assert result.results[0].scores == {} + assert "valid Score object" in result.results[0].metadata["scorer_errors"]["scorer"] + + @pytest.mark.asyncio async def test_run_evaluator_normalizes_list_of_dict_scores(): data = [ diff --git a/py/src/braintrust/type_tests/test_dict_scorers.py b/py/src/braintrust/type_tests/test_dict_scorers.py new file mode 100644 index 00000000..837e6325 --- /dev/null +++ b/py/src/braintrust/type_tests/test_dict_scorers.py @@ -0,0 +1,100 @@ +"""Type-check and runtime coverage for dictionary scorer returns.""" + +from collections.abc import Sequence + +import pytest +from braintrust import NamedScoreDict, ScoreDict +from braintrust.framework import Eval, EvalAsync, EvalCase, EvalScorer, OneOrMoreScores +from braintrust.score import Score, ScoreLike + + +LegacyScores = float | int | bool | None | ScoreLike | Sequence[ScoreLike] + + +@pytest.mark.parametrize( + "value,expected_scores", + [ + (1.0, {"scorer": 1.0}), + (0, {"scorer": 0}), + (True, {"scorer": True}), + (False, {"scorer": False}), + (None, {"scorer": None}), + (Score(name="named", score=0.5), {"named": 0.5}), + ([Score(name="named", score=0.5)], {"named": 0.5}), + ((Score(name="named", score=0.5),), {"named": 0.5}), + ], +) +def test_eval_accepts_legacy_return_annotation(value: LegacyScores, expected_scores: dict[str, float | None]) -> None: + def scorer(input: str, output: str, expected: str | None = None) -> LegacyScores: + return value + + typed_scorer: EvalScorer[str, str, str] = scorer + result = Eval( + "test-legacy-scorer", + data=[EvalCase(input="hello", expected="hello")], + task=lambda input: input, + scores=[typed_scorer], + no_send_logs=True, + ) + + assert result.results[0].scores == expected_scores + + +def test_eval_accepts_single_dict_scorer() -> None: + def scorer(input: str, output: str, expected: str | None = None) -> ScoreDict: + return {"score": 1.0, "metadata": {"reason": "Matches"}} + + typed_scorer: EvalScorer[str, str, str] = scorer + result = Eval( + "test-dict-scorers", + data=[EvalCase(input="hello", expected="hello")], + task=lambda input: input, + scores=[typed_scorer], + no_send_logs=True, + ) + + assert result.results[0].scores == {"scorer": 1.0} + + +@pytest.mark.asyncio +async def test_eval_async_accepts_single_dict_scorer() -> None: + async def scorer(input: str, output: str, expected: str | None = None) -> OneOrMoreScores: + return {"score": None} + + typed_scorer: EvalScorer[str, str, str] = scorer + result = await EvalAsync( + "test-dict-scorers", + data=[EvalCase(input="hello", expected="hello")], + task=lambda input: input, + scores=[typed_scorer], + no_send_logs=True, + ) + + assert result.results[0].scores == {"scorer": None} + + +def test_eval_accepts_named_dict_scores() -> None: + def scorer(input: str, output: str, expected: str | None = None) -> list[NamedScoreDict]: + return [{"name": "match", "score": 1.0}, {"name": "quality", "score": 0.5}] + + result = Eval( + "test-dict-scorers", + data=[EvalCase(input="hello", expected="hello")], + task=lambda input: input, + scores=[scorer], + no_send_logs=True, + ) + + assert result.results[0].scores == {"match": 1.0, "quality": 0.5} + + +def test_eval_accepts_inline_dict_scorer() -> None: + result = Eval( + "test-dict-scorers", + data=[EvalCase(input="hello", expected="hello")], + task=lambda input: input, + scores=[lambda input, output, expected: ScoreDict(score=1.0)], + no_send_logs=True, + ) + + assert result.results[0].scores == {"scorer_0": 1.0}