From 7846dfc38cfd02a0a9bd2c1a5a99d698ed0c360b Mon Sep 17 00:00:00 2001 From: paultancre-bt Date: Mon, 7 Sep 2026 10:21:30 +0100 Subject: [PATCH 1/2] fix(pydantic_ai): capture Anthropic/Google reasoning tokens `_extract_response_metrics` only read `details["reasoning_tokens"]`, which is the key pydantic-ai uses for OpenAI. Anthropic reasoning is stashed under `thinking_tokens` and Google under `thoughts_tokens`, so `completion_reasoning_tokens` was silently empty for those providers (reported for Anthropic in a customer trace set: 16,587/16,587 llm spans with no reasoning metric, all provider=anthropic). Read whichever of the three normalized keys is present. Verified end-to-end by driving pydantic-ai's real `_map_usage` from an Anthropic response with thinking tokens through the extractor. Co-Authored-By: Claude Opus 4.8 --- .../test_pydantic_ai_integration.py | 30 +++++++++++++++++++ .../integrations/pydantic_ai/tracing.py | 13 +++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index ee281d87..79f323ef 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -2212,6 +2212,36 @@ def calc(a: int, b: int) -> int: assert "b" in tool["parameters"]["properties"] +@pytest.mark.parametrize( + "details_key", + [ + "reasoning_tokens", # OpenAI + "thinking_tokens", # Anthropic + "thoughts_tokens", # Google + ], +) +def test_reasoning_tokens_extraction_provider_keys(details_key): + """pydantic_ai stashes the reasoning-token count under a provider-specific key: + OpenAI "reasoning_tokens", Anthropic "thinking_tokens", Google "thoughts_tokens". + All three must surface as `completion_reasoning_tokens` (previously only OpenAI's + key was read, silently dropping Anthropic/Google reasoning). + """ + from types import SimpleNamespace + + from braintrust.integrations.pydantic_ai.tracing import _extract_response_metrics + from pydantic_ai.usage import RequestUsage + + usage = RequestUsage(input_tokens=10, output_tokens=20, details={details_key: 128}) + response = SimpleNamespace(parts=[], usage=usage) + + metrics = _extract_response_metrics(response, start_time=1.0, end_time=2.0) + + assert metrics is not None + # pylint: disable=unsupported-membership-test,unsubscriptable-object + assert metrics["completion_reasoning_tokens"] == 128.0 + # pylint: enable=unsupported-membership-test,unsubscriptable-object + + def test_explicit_toolsets_kwarg_in_input(): """Test that explicitly passed toolsets kwarg goes to input (not just metadata).""" from braintrust.integrations.pydantic_ai.tracing import _build_agent_input_and_metadata diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index a87620ea..e395ee61 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -1392,9 +1392,20 @@ def _extract_response_metrics( metrics["completion_audio_tokens"] = float(usage.output_audio_tokens) # RequestUsage.details is a dict; providers stash reasoning_tokens/cached_tokens here. + # The reasoning-token count lives under a provider-specific key: OpenAI uses + # "reasoning_tokens", Anthropic "thinking_tokens", Google "thoughts_tokens". All three + # are the same normalized quantity (a subset of the output tokens), so read whichever + # is present. details = getattr(usage, "details", None) if isinstance(details, dict): - reasoning = details.get("reasoning_tokens") + reasoning = next( + ( + details[key] + for key in ("reasoning_tokens", "thinking_tokens", "thoughts_tokens") + if details.get(key) is not None + ), + None, + ) if reasoning is not None: metrics["completion_reasoning_tokens"] = float(reasoning) cached = details.get("cached_tokens") From 3e225d8655ad19af4c0abb1307688ba4930b1460 Mon Sep 17 00:00:00 2001 From: paultancre-bt Date: Mon, 7 Sep 2026 10:39:09 +0100 Subject: [PATCH 2/2] test(pydantic_ai): cassette-backed Anthropic reasoning-token coverage Address review feedback: the synthetic parametrized test verifies the extractor reads each provider key, but not that pydantic-ai actually populates thinking_tokens from a real Anthropic response. Add a cassette-backed test that runs a real Agent against Anthropic (extended thinking enabled) through pydantic-ai's own _map_usage, asserting the leaf chat span surfaces completion_reasoning_tokens. Keeps the synthetic test as supplemental coverage (per AGENTS.md VCR-first rule). Co-Authored-By: Claude Opus 4.8 --- ..._agent_run_anthropic_reasoning_tokens.yaml | 68 +++++++++++++++++++ .../test_pydantic_ai_integration.py | 45 ++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_run_anthropic_reasoning_tokens.yaml diff --git a/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_run_anthropic_reasoning_tokens.yaml b/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_run_anthropic_reasoning_tokens.yaml new file mode 100644 index 00000000..9bbcfd4f --- /dev/null +++ b/py/src/braintrust/integrations/pydantic_ai/cassettes/latest/test_agent_run_anthropic_reasoning_tokens.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"max_tokens":2048,"messages":[{"role":"user","content":[{"text":"What + is 17 * 23? Think it through, then give the number.","type":"text"}]}],"model":"claude-haiku-4-5-20251001","stream":false,"thinking":{"type":"enabled","budget_tokens":1024}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '244' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - pydantic-ai/2.36.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.2.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.15 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages?beta=true + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CeorTzoNmBZLscy4jhvVS","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"17 + * 23\n\nLet me break this down:\n17 * 23 = 17 * (20 + 3)\n= 17 * 20 + 17 * + 3\n= 340 + 51\n= 391\n\nLet me verify: \n17 * 23\n= (20 - 3) * 23\n= 20 * + 23 - 3 * 23\n= 460 - 69\n= 391\n\nYes, 391 is correct.","signature":"Er0DCpoBCBEYAipAfQOcVMh+fN1kqKgyYniNFmLoXQndf7+wItJBZdo15tWSaYQloTdiBYEPUjjfVb5BWuTi7Y2SC01sXMYmfZe+sTIZY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMTgAQgh0aGlua2luZ1okMjc3OTY2NjgtNzM1MS00MGFjLWFjYzQtMDI0YWVlODk5NWE1qAG7i/rUBhIMwNYhm9izIza3IBM+Ggw3Yl78vAm4cQvUy1IiMC+l0Z2l9ca6Ed9mQVMPsQkx8I/TEWhrLmvfeEZFefFnfDsmfJz79ZHIg+KGNzVXtirPAaJFPsSgcn6FdRadilHlhlgKCcvbQuyaHEDd4iYQb/UVnFK5xwpyuFJDw9qoWmghP5LmkAdbKfj/iA+2mnJDE69ZVd//dHtVu6Y58bH9B3Tbwku46gf/0rru7m6dd6k0DztvAqT5wSQ4KUV4iGdID1hNlo+Qj3QXYAXrJkQCnWPjlVdqCXH2z02uR2B2fSC/KDCcUQ2lIJBYCNmYPVBi8BmJ4X8yC8/XoewSy26hKUMM3oLhnQ5k28UB1HMe2i/F0Uur7oeExShBTJXLWfPnkhgB"},{"type":"text","text":"Let + me work through this:\n\n17 * 23 = 17 * (20 + 3)\n= (17 * 20) + (17 * 3)\n= + 340 + 51\n= **391**"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":54,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":186,"output_tokens_details":{"thinking_tokens":125},"service_tier":"standard","inference_geo":"not_available"}}' + headers: + connection: + - keep-alive + content-type: + - application/json + date: + - Mon, 07 Sep 2026 09:36:59 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index 79f323ef..d2fb0074 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -3,6 +3,7 @@ # pyright: reportUnknownParameterType=false # pyright: reportPrivateUsage=false import inspect +import os import time import pytest @@ -2212,6 +2213,50 @@ def calc(a: int, b: int) -> int: assert "b" in tool["parameters"]["properties"] +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path"]) +@pytest.mark.asyncio +async def test_agent_run_anthropic_reasoning_tokens(memory_logger): + """Real Anthropic extended-thinking response through pydantic-ai's own usage mapping. + + pydantic-ai stashes the reasoning count under ``details["thinking_tokens"]`` for + Anthropic (vs ``reasoning_tokens`` for OpenAI); the extractor must surface it as + ``completion_reasoning_tokens``. This exercises the real provider mapping via a + checked-in cassette; ``test_reasoning_tokens_extraction_provider_keys`` is the + supplemental synthetic coverage. + """ + if os.environ.get("BRAINTRUST_TEST_PACKAGE_VERSION") != "latest": + pytest.skip("Anthropic extended-thinking usage requires the latest pydantic-ai cassette") + + import anthropic + from pydantic_ai.models.anthropic import AnthropicModel, AnthropicModelSettings + from pydantic_ai.providers.anthropic import AnthropicProvider + + assert not memory_logger.pop() + + model = AnthropicModel( + "claude-haiku-4-5-20251001", + provider=AnthropicProvider(anthropic_client=anthropic.AsyncAnthropic()), + ) + agent = Agent( + model, + model_settings=AnthropicModelSettings( + max_tokens=2048, + anthropic_thinking={"type": "enabled", "budget_tokens": 1024}, + ), + ) + + result = await agent.run("What is 17 * 23? Think it through, then give the number.") + assert "391" in str(result.output) + + spans = memory_logger.pop() + chat_span = next((s for s in spans if "chat" in s["span_attributes"]["name"]), None) + assert chat_span is not None, "chat span not found" + assert chat_span["metadata"]["provider"] == "anthropic" + # pylint: disable=unsupported-membership-test,unsubscriptable-object + assert chat_span["metrics"]["completion_reasoning_tokens"] > 0 + # pylint: enable=unsupported-membership-test,unsubscriptable-object + + @pytest.mark.parametrize( "details_key", [