diff --git a/py/src/braintrust/integrations/anthropic/_utils.py b/py/src/braintrust/integrations/anthropic/_utils.py index 12db0e2f2..e6c38273b 100644 --- a/py/src/braintrust/integrations/anthropic/_utils.py +++ b/py/src/braintrust/integrations/anthropic/_utils.py @@ -91,6 +91,13 @@ def extract_anthropic_usage( if cache_creation_breakdown: metrics.pop("prompt_cache_creation_tokens", None) + output_tokens_details = _try_to_dict(usage.get("output_tokens_details")) + if include_output and output_tokens_details is not None: + # Anthropic reports extended-thinking usage as a subset of output_tokens, under the + # nested output_tokens_details.thinking_tokens field. Surface it as the normalized + # completion_reasoning_tokens metric (matching the OpenAI/pydantic-ai integrations). + _set_numeric_metric(metrics, "completion_reasoning_tokens", output_tokens_details.get("thinking_tokens")) + server_tool_use = _try_to_dict(usage.get("server_tool_use")) if server_tool_use is not None: for source_name, value in server_tool_use.items(): diff --git a/py/src/braintrust/integrations/anthropic/cassettes/0.48.0/test_anthropic_messages_create_reasoning_tokens_metrics.yaml b/py/src/braintrust/integrations/anthropic/cassettes/0.48.0/test_anthropic_messages_create_reasoning_tokens_metrics.yaml new file mode 100644 index 000000000..002550229 --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/0.48.0/test_anthropic_messages_create_reasoning_tokens_metrics.yaml @@ -0,0 +1,113 @@ +interactions: +- request: + body: '{"max_tokens":2048,"messages":[{"role":"user","content":"What is 17 * 23? + Think it through, then give the number."}],"model":"claude-haiku-4-5-20251001","thinking":{"type":"enabled","budget_tokens":1024}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '204' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.48.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 0.48.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011Cep4FvAdw5k9kFJqR5trP","type":"message","role":"assistant","content":[{"type":"thinking","thinking":"Let + me calculate 17 * 23.\n\nI can break this down:\n17 * 23 = 17 * (20 + 3)\n= + 17 * 20 + 17 * 3\n= 340 + 51\n= 391\n\nLet me verify this another way:\n17 + * 23 = (20 - 3) * 23\n= 20 * 23 - 3 * 23\n= 460 - 69\n= 391\n\nYes, that''s + correct.","signature":"Et4DCpoBCBEYAipAjRXpS/Xogu4z9JT1WNaqulh8un2XjgYshUyW62VC4OrbP71M+2O94HdPIAHCHwaXBSIIoaIEwjI7fFwhxbcdezIZY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMTgAQgh0aGlua2luZ1okMjc3OTY2NjgtNzM1MS00MGFjLWFjYzQtMDI0YWVlODk5NWE1qAH50/rUBhIMshMqDkSSKZFHM0PfGgxVK+BEEKCBAZKWsXEiMItCFjLNfYjkvT7BEERaQ4kaCA853Aa40E80dS1EDMNX9TqS5nyvVbA6swRCIlFF8CrwAe4QWNw6X4IKk4glFsus0MK8ZU7JUE+bdCueBUScZEWzX1MaZnweSdw+tZm9oRnJiJDNEjIMCXS17T0ua3RlAnMypfcwg8T/Rk7oYwYux5LnEwOfNTLAZvt4r01OqFTvfwn/hDDC7JvgsIA6Ol101LzqsO1lom9K6idbPk+FunOkk7peVoI8U5KFQlh8jBxLLAcZhPXDuuCoVEKwvB1e6/SbjPk9JNo1hO3hhXzCwxECK1NInWBKYZZz54Q1Bo/FfbLl4x8g+fJAiqdQgWzG4jlkmLmd0KMcF3ikqSgxMfY+MQLgRvfRt79xy8r5YhF7rRgB"},{"type":"text","text":"Let + me work through this:\n\n17 * 23 = 17 * (20 + 3)\n= (17 * 20) + (17 * 3)\n= + 340 + 51\n= **391**"}],"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":54,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":191,"output_tokens_details":{"thinking_tokens":130},"service_tier":"standard","inference_geo":"not_available"}}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-09-07T12:11:36Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-09-07T12:11:37Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-09-07T12:11:35Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-09-07T12:11:36Z' + anthropic-workspace-id: + - wrkspc_01J8hau4yJ2SKvtbeP9drm9A + cf-cache-status: + - DYNAMIC + cf-ray: + - a3759ded3f1bc96a-IAD + connection: + - keep-alive + content-length: + - '1536' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Mon, 07 Sep 2026 12:11:37 GMT + request-id: + - req_011Cep4FuaQrAQvogvgT22gJ + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-dfd54e78541d06a1123d9af064aec955-51ee7da98cfd57f9-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/cassettes/0.48.0/test_anthropic_messages_stream_reasoning_tokens_metrics.yaml b/py/src/braintrust/integrations/anthropic/cassettes/0.48.0/test_anthropic_messages_stream_reasoning_tokens_metrics.yaml new file mode 100644 index 000000000..0be1aaeab --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/0.48.0/test_anthropic_messages_stream_reasoning_tokens_metrics.yaml @@ -0,0 +1,132 @@ +interactions: +- request: + body: '{"max_tokens":2048,"messages":[{"role":"user","content":"What is 17 * 23? + Think it through, then give the number."}],"model":"claude-haiku-4-5-20251001","stream":true,"thinking":{"type":"enabled","budget_tokens":1024}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '218' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.48.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 0.48.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011Cep4CYQrfbDgk2SQV7Pg6\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":54,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} + \ }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"} + \ }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: + {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Let\"} + \ }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" + me calculate 17 * 23.\\n\\nI can break this down:\\n17 * 23 = 17 * (20 + 3)\\n= + 17 * 20 + 17\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" + * 3\\n= 340 + 51\\n= 391\\n\\nAlternatively:\\n17 * 23 = (10 + 7) * 23\\n= + 10 * \"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"23 + + 7 * 23\\n= 230 + 161\\n= 391\\n\\nLet me verify once more:\\n17 * 23\\n= + 17 * 20 + 17\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" + * 3\\n= 340 + 51\\n= 391\\n\\nYes, 391 is correct.\"} }\n\nevent: + content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EpQECpoBCBEYAipANHewmm1S9pfqd1f1NsPz65CoWBhVOSwzlz/tlzNmt6Gy8dLhvJIBWf+6g+1OtMYk6Y0MChVFM+UgtHHETYe5uTIZY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMTgAQgh0aGlua2luZ1okMjc3OTY2NjgtNzM1MS00MGFjLWFjYzQtMDI0YWVlODk5NWE1qAHL0/rUBhIMDR1e8EblmM3G5OWPGgw7w7Gd652yQDx11zIiMIx53xS7pyMhznnwemeMXxcCpDElb+NGKf7IsSyk11z5GQV3Ai4lekthU4Dj0NqOYyqmAokwZrvXTihY4pgUsBYv5debbZRd/SwxB0m5YVmFx5cACZ59uK62Pjtw0rtEONiwuCzVCNpoJIPLzHrxPCLEQBhmBAyOyMEK+yq1H+de7FHKc7vHqdE9GBbjYQZyHvHG/X7mtUNU43DT2G59r2XqGV2T+WMWTEckg0l5SMVIwmu+RBEV/Tx5rNdzCPaVwucgiir28W3lqIIRJfto072ISzAf2t0tRck/WE+6Uv76qjHY95OTTmBNxqCtCnD6NL+RK6oeRTCfL4mYbyWol0pjCnu+8Mu2D231pq3Xy4BUac2x54H1i5uw8EbtsJ3bTwolgOQMP/L5cyB7Ea6hAhAqeuJKmB0VyHZe/CfPbxBBwrXt126jv/SHuARq8ehpEmfWYT2FAVFbfRgB\"} + \ }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 + \ }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"} + \ }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"# + Calculating 17 \xD7 23\\n\\nBreaking\"} }\n\nevent: content_block_delta\ndata: + {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\" + it down:\\n- 17 \xD7 23 = 17 \xD7 (20 + 3)\\n- = (17 \xD7 20) + (17 \xD7 3)\\n- + = 340 \"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"+ + 51\\n- = **391**\\n\\nThe answer is **391**.\"} }\n\nevent: content_block_stop\ndata: + {\"type\":\"content_block_stop\",\"index\":1}\n\nevent: message_delta\ndata: + {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":54,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":250,\"output_tokens_details\":{\"thinking_tokens\":169}} + \ }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-09-07T12:10:50Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-09-07T12:10:50Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-09-07T12:10:50Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-09-07T12:10:50Z' + anthropic-workspace-id: + - wrkspc_01J8hau4yJ2SKvtbeP9drm9A + cache-control: + - no-cache + cf-cache-status: + - DYNAMIC + cf-ray: + - a3759ccebeba4c80-IAD + connection: + - keep-alive + content-length: + - '3650' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - text/event-stream; charset=utf-8 + date: + - Mon, 07 Sep 2026 12:10:50 GMT + request-id: + - req_011Cep4CXdj2dbuiKhQRa6T1 + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-91d778ba9a76bedf9b72507b165b7243-44d7f2c5c22c3c82-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_reasoning_tokens_metrics.yaml b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_reasoning_tokens_metrics.yaml new file mode 100644 index 000000000..1d60b5030 --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_create_reasoning_tokens_metrics.yaml @@ -0,0 +1,69 @@ +interactions: +- request: + body: '{"max_tokens":2048,"messages":[{"role":"user","content":"What is 17 * 23? + Think it through, then give the number."}],"model":"claude-haiku-4-5-20251001","thinking":{"type":"enabled","budget_tokens":1024}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '204' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 1.2.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.2.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.15 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: "{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011CeoqtD3dWGwJ1229TnnFD\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[{\"type\":\"thinking\",\"thinking\":\"17 + * 23\\n\\nLet me break this down:\\n17 * 23 = 17 * (20 + 3)\\n= 17 * 20 + + 17 * 3\\n= 340 + 51\\n= 391\\n\\nLet me double-check:\\n17 * 20 = 340 ✓\\n17 + * 3 = 51 ✓\\n340 + 51 = 391 ✓\\n\\nAlternatively:\\n17 * 23 = (10 + + 7) * 23\\n= 10 * 23 + 7 * 23\\n= 230 + 161\\n= 391 ✓\",\"signature\":\"EvYDCpoBCBEYAipAJUDXcciCV/oALWk53KOqQ43mNIbhG1QWcimrkKC9UtYmzjGqPkUAY1yaP3URbyLFmY2GtC2L4DyY/bHDYtghlzIZY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMTgAQgh0aGlua2luZ1okMjc3OTY2NjgtNzM1MS00MGFjLWFjYzQtMDI0YWVlODk5NWE1qAHxh/rUBhIMNZJuVeYYxJ2RsKIrGgycuj7DxA2X5mSez7AiMAVKuQ01okNXZPxQL+fnZDp8HaivH4MGgb1aYDygkv98bXje2HFwREsLluc81yimZyqIAiF5KqKOkCSAWX0dgug2D7e0SUwMql070+i+6dJ1Kx9b518BWF0gWOFrHlCZ39VptVamutI7P+Gv1aXd8W8eNANN222vGX5D5nqRz/rY4Z+lLld4mfr4jUrMiMUenJpOaiO33ZdQ10jPmn0OZVAfUHcQByJQkbrP1Hqzr77ahsPchSesoYYD83smavcj1XMLxu2VXZ6NKO1gnxSjJ6nMCfC44CZBim/3k2qYz5ABIiJqLPA44okcRKxrF5qWF3Nc/2bwQEs+f/csAAMAKvMjP47pibT7tFvOYxhS71YI1BLBU4ka5SxF0u/NxqC3YOAgwv4Wc6UcvCGZSsvRsQxvwumeAr+H0a/6GRgB\"},{\"type\":\"text\",\"text\":\"17 + * 23:\\n\\nI'll break this into parts:\\n- 17 * 20 = 340\\n- 17 * 3 = 51\\n- + 340 + 51 = 391\\n\\n**391**\"}],\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":54,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":227,\"output_tokens_details\":{\"thinking_tokens\":168},\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}" + headers: + connection: + - keep-alive + content-type: + - application/json + date: + - Mon, 07 Sep 2026 09:29:21 GMT + strict-transport-security: + - max-age=31536000; includeSubDomains + transfer-encoding: + - chunked + vary: + - accept-encoding + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_stream_reasoning_tokens_metrics.yaml b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_stream_reasoning_tokens_metrics.yaml new file mode 100644 index 000000000..0be1aaeab --- /dev/null +++ b/py/src/braintrust/integrations/anthropic/cassettes/latest/test_anthropic_messages_stream_reasoning_tokens_metrics.yaml @@ -0,0 +1,132 @@ +interactions: +- request: + body: '{"max_tokens":2048,"messages":[{"role":"user","content":"What is 17 * 23? + Think it through, then give the number."}],"model":"claude-haiku-4-5-20251001","stream":true,"thinking":{"type":"enabled","budget_tokens":1024}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - '2023-06-01' + connection: + - keep-alive + content-length: + - '218' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - Anthropic/Python 0.48.0 + x-stainless-arch: + - x64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - Linux + x-stainless-package-version: + - 0.48.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.6 + x-stainless-timeout: + - NOT_GIVEN + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-haiku-4-5-20251001\",\"id\":\"msg_011Cep4CYQrfbDgk2SQV7Pg6\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":54,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":0,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}} + \ }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"} + \ }\n\nevent: ping\ndata: {\"type\": \"ping\"}\n\nevent: content_block_delta\ndata: + {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Let\"} + \ }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" + me calculate 17 * 23.\\n\\nI can break this down:\\n17 * 23 = 17 * (20 + 3)\\n= + 17 * 20 + 17\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" + * 3\\n= 340 + 51\\n= 391\\n\\nAlternatively:\\n17 * 23 = (10 + 7) * 23\\n= + 10 * \"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"23 + + 7 * 23\\n= 230 + 161\\n= 391\\n\\nLet me verify once more:\\n17 * 23\\n= + 17 * 20 + 17\"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" + * 3\\n= 340 + 51\\n= 391\\n\\nYes, 391 is correct.\"} }\n\nevent: + content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EpQECpoBCBEYAipANHewmm1S9pfqd1f1NsPz65CoWBhVOSwzlz/tlzNmt6Gy8dLhvJIBWf+6g+1OtMYk6Y0MChVFM+UgtHHETYe5uTIZY2xhdWRlLWhhaWt1LTQtNS0yMDI1MTAwMTgAQgh0aGlua2luZ1okMjc3OTY2NjgtNzM1MS00MGFjLWFjYzQtMDI0YWVlODk5NWE1qAHL0/rUBhIMDR1e8EblmM3G5OWPGgw7w7Gd652yQDx11zIiMIx53xS7pyMhznnwemeMXxcCpDElb+NGKf7IsSyk11z5GQV3Ai4lekthU4Dj0NqOYyqmAokwZrvXTihY4pgUsBYv5debbZRd/SwxB0m5YVmFx5cACZ59uK62Pjtw0rtEONiwuCzVCNpoJIPLzHrxPCLEQBhmBAyOyMEK+yq1H+de7FHKc7vHqdE9GBbjYQZyHvHG/X7mtUNU43DT2G59r2XqGV2T+WMWTEckg0l5SMVIwmu+RBEV/Tx5rNdzCPaVwucgiir28W3lqIIRJfto072ISzAf2t0tRck/WE+6Uv76qjHY95OTTmBNxqCtCnD6NL+RK6oeRTCfL4mYbyWol0pjCnu+8Mu2D231pq3Xy4BUac2x54H1i5uw8EbtsJ3bTwolgOQMP/L5cyB7Ea6hAhAqeuJKmB0VyHZe/CfPbxBBwrXt126jv/SHuARq8ehpEmfWYT2FAVFbfRgB\"} + \ }\n\nevent: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0 + \ }\n\nevent: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"} + \ }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"# + Calculating 17 \xD7 23\\n\\nBreaking\"} }\n\nevent: content_block_delta\ndata: + {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\" + it down:\\n- 17 \xD7 23 = 17 \xD7 (20 + 3)\\n- = (17 \xD7 20) + (17 \xD7 3)\\n- + = 340 \"} }\n\nevent: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"+ + 51\\n- = **391**\\n\\nThe answer is **391**.\"} }\n\nevent: content_block_stop\ndata: + {\"type\":\"content_block_stop\",\"index\":1}\n\nevent: message_delta\ndata: + {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":54,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":250,\"output_tokens_details\":{\"thinking_tokens\":169}} + \ }\n\nevent: message_stop\ndata: {\"type\":\"message_stop\" }\n\n" + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-09-07T12:10:50Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-09-07T12:10:50Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-09-07T12:10:50Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-09-07T12:10:50Z' + anthropic-workspace-id: + - wrkspc_01J8hau4yJ2SKvtbeP9drm9A + cache-control: + - no-cache + cf-cache-status: + - DYNAMIC + cf-ray: + - a3759ccebeba4c80-IAD + connection: + - keep-alive + content-length: + - '3650' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - text/event-stream; charset=utf-8 + date: + - Mon, 07 Sep 2026 12:10:50 GMT + request-id: + - req_011Cep4CXdj2dbuiKhQRa6T1 + server: + - cloudflare + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-91d778ba9a76bedf9b72507b165b7243-44d7f2c5c22c3c82-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/anthropic/test_anthropic.py b/py/src/braintrust/integrations/anthropic/test_anthropic.py index 6c25c4a04..55b531dde 100644 --- a/py/src/braintrust/integrations/anthropic/test_anthropic.py +++ b/py/src/braintrust/integrations/anthropic/test_anthropic.py @@ -15,6 +15,7 @@ from braintrust.integrations.anthropic import AnthropicIntegration, wrap_anthropic from braintrust.integrations.anthropic._utils import _try_to_dict, extract_anthropic_usage from braintrust.integrations.anthropic.tracing import ( + TracedMessageStream, _get_input_from_kwargs, _get_metadata_from_kwargs, _log_message_to_span, @@ -448,6 +449,158 @@ def test_anthropic_messages_create_prompt_cache_1h_metrics(memory_logger): ) +@pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path", "body"]) +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "mode,vcr_cassette_name", + [ + ("create", "test_anthropic_messages_create_reasoning_tokens_metrics"), + ("create_stream", "test_anthropic_messages_stream_reasoning_tokens_metrics"), + ("stream", "test_anthropic_messages_stream_reasoning_tokens_metrics"), + ("text_stream", "test_anthropic_messages_stream_reasoning_tokens_metrics"), + ], + ids=["create", "create_stream", "stream", "text_stream"], +) +async def test_anthropic_messages_reasoning_tokens_metrics( + memory_logger, vcr_cassette, vcr_cassette_name, mode, is_async +): + client = wrap_anthropic(_get_async_client() if is_async else _get_client()) + params = { + "model": LATEST_MODEL, + "max_tokens": 2048, + "thinking": {"type": "enabled", "budget_tokens": 1024}, + "messages": [{"role": "user", "content": "What is 17 * 23? Think it through, then give the number."}], + } + events = [] + text = "" + if mode == "create": + response = await client.messages.create(**params) if is_async else client.messages.create(**params) + text = "".join(block.text for block in response.content if block.type == "text") + elif mode == "create_stream": + if is_async: + stream = await client.messages.create(**params, stream=True) + events = [event async for event in stream] + else: + with client.messages.create(**params, stream=True) as stream: + events = list(stream) + elif is_async: + async with client.messages.stream(**params) as stream: + if mode == "text_stream": + text = "".join([chunk async for chunk in stream.text_stream]) + else: + events = [event async for event in stream] + else: + with client.messages.stream(**params) as stream: + if mode == "text_stream": + text = "".join(stream.text_stream) + else: + events = list(stream) + + if events: + assert events[0].type == "message_start" + assert events[-1].type == "message_stop" + text = "".join( + event.delta.text + for event in events + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ) + assert "391" in text + + # Read expected usage from the wire, independently of the SDK's accumulator: + # older SDKs retain unknown fields on events but discard them from snapshots. + body = vcr_cassette.responses[0]["body"]["string"] + if isinstance(body, bytes): + body = body.decode() + if mode == "create": + usage = json.loads(body)["usage"] + else: + usage = {} + for line in body.splitlines(): + if line.startswith("data: "): + event = json.loads(line[6:]) + if event["type"] == "message_start": + usage.update(event["message"]["usage"]) + elif event["type"] == "message_delta": + usage.update(event["usage"]) + + thinking_tokens = usage["output_tokens_details"]["thinking_tokens"] + assert thinking_tokens > 0 + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["name"] == f"anthropic.messages.{'create' if mode == 'create' else 'stream'}" + assert span["span_attributes"]["type"] == "llm" + assert span["context"]["span_origin"]["instrumentation"]["name"] == "anthropic-auto" + assert span["metadata"]["model"] == LATEST_MODEL + assert span["metadata"]["provider"] == "anthropic" + assert span["input"] == params["messages"] + assert any(block["type"] == "thinking" for block in span["output"]["content"]) + assert "".join(block["text"] for block in span["output"]["content"] if block["type"] == "text") == text + metrics = span["metrics"] + assert metrics["completion_reasoning_tokens"] == thinking_tokens + assert metrics["completion_tokens"] == usage["output_tokens"] + assert metrics["prompt_tokens"] == ( + usage["input_tokens"] + usage["cache_creation_input_tokens"] + usage["cache_read_input_tokens"] + ) + assert metrics["tokens"] == metrics["prompt_tokens"] + usage["output_tokens"] + assert metrics["time_to_first_token"] >= 0 + + +@pytest.mark.parametrize("final_thinking_tokens", [None, 0, 17]) +def test_anthropic_stream_reasoning_tokens_are_cumulative(memory_logger, final_thinking_tokens): + # Supplemental coverage for repeated/omitted usage fields and explicit zero, + # which the real-response regression above cannot deterministically request. + events = [ + anthropic.types.RawMessageStartEvent.model_validate( + { + "type": "message_start", + "message": { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": LATEST_MODEL, + "content": [], + "usage": { + "input_tokens": 11, + "cache_read_input_tokens": 3, + "cache_creation_input_tokens": 2, + "output_tokens": 1, + "output_tokens_details": {"thinking_tokens": 1}, + }, + }, + } + ) + ] + for output_tokens, thinking_tokens in [(10, 5), (30, final_thinking_tokens)]: + events.append( + anthropic.types.RawMessageDeltaEvent.model_validate( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": { + "output_tokens": output_tokens, + "output_tokens_details": ( + {"thinking_tokens": thinking_tokens} if thinking_tokens is not None else None + ), + }, + } + ) + ) + original_events = [event.model_dump() for event in events] + with logger.start_span(name="reasoning usage") as span: + stream = TracedMessageStream(iter(events), span, time.time()) + assert list(stream) == events + stream._log_final_message() + + assert [event.model_dump() for event in events] == original_events + metrics = memory_logger.pop()[0]["metrics"] + assert metrics["completion_reasoning_tokens"] == (5 if final_thinking_tokens is None else final_thinking_tokens) + assert metrics["prompt_tokens"] == 16 + assert metrics["completion_tokens"] == 30 + assert metrics["tokens"] == 46 + + @pytest.mark.vcr(match_on=["method", "scheme", "host", "port", "path"]) def test_anthropic_messages_create_with_image_attachment_input(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/anthropic/tracing.py b/py/src/braintrust/integrations/anthropic/tracing.py index f5c1a307f..1ebf4e9eb 100644 --- a/py/src/braintrust/integrations/anthropic/tracing.py +++ b/py/src/braintrust/integrations/anthropic/tracing.py @@ -131,10 +131,7 @@ async def async_stream(): span.log(error=e) raise finally: - msg = traced_stream._get_final_traced_message() - if msg: - ttft = traced_stream._get_time_to_first_token() - _log_message_to_span(msg, span, time_to_first_token=ttft) + traced_stream._log_final_message() span.end() return async_stream() @@ -690,10 +687,7 @@ def __exit__(self, exc_type, exc_value, traceback): def __close(self, exc_type, exc_value, traceback): tms = self.__traced_message_stream - msg = tms._get_final_traced_message() - if msg: - ttft = tms._get_time_to_first_token() - _log_message_to_span(msg, self.__span, time_to_first_token=ttft) + tms._log_final_message() if exc_type: log_exc_info_to_span(self.__span, exc_type, exc_value, traceback) self.__span.end() @@ -711,11 +705,14 @@ def __init__(self, msg_stream, span, request_start_time: float): self.__request_start_time = request_start_time self.__time_to_first_token: float | None = None - def _get_final_traced_message(self): - return self.__snapshot - - def _get_time_to_first_token(self): - return self.__time_to_first_token + def _log_final_message(self): + if self.__snapshot: + _log_message_to_span( + self.__snapshot, + self.__span, + time_to_first_token=self.__time_to_first_token, + metrics_override=self.__metrics, + ) def __await__(self): return self.__msg_stream.__await__() @@ -762,6 +759,14 @@ def __process_message(self, m): self.__snapshot = accumulate_event(event=m, current_snapshot=self.__snapshot) + if m.type == "message_delta": + # Anthropic <0.122.0 drops output_tokens_details when accumulating + # events. Keep the reported reasoning count separately, without + # mutating SDK objects or changing completion/prompt token totals. + metrics, _ = extract_anthropic_usage(m.usage) + if "completion_reasoning_tokens" in metrics: + self.__metrics["completion_reasoning_tokens"] = metrics["completion_reasoning_tokens"] + class TracedManagedAgentsEventStream(Wrapper): def __init__(self, stream, span): @@ -1499,9 +1504,13 @@ def _message_output(message, *, include_parsed_output: bool = False): return output or None -def _log_message_to_span(message, span, time_to_first_token: float | None = None, *, include_parsed_output=False): +def _log_message_to_span( + message, span, time_to_first_token: float | None = None, *, include_parsed_output=False, metrics_override=None +): usage = getattr(message, "usage", {}) metrics, metadata = extract_anthropic_usage(usage) + if metrics_override: + metrics.update(metrics_override) if time_to_first_token is not None: metrics["time_to_first_token"] = time_to_first_token