feat!: Add BRAINTRUST_CAPTURE_ATTACHMENTS env var option and captureAttachments logger option for all integrations

.changeset/light-plants-return.md

@@ -0,0 +1,7 @@
+---
+"@braintrust/deepseek-harness": major
+"braintrust": major
+"@braintrust/openai-agents": minor
+---
+
+feat!: Add `BRAINTRUST_CAPTURE_ATTACHMENTS` env var option and `captureAttachments` logger option for all integrations

e2e/helpers/attachment-capture-assertions.ts

@@ -0,0 +1,73 @@
+import { expect, test } from "vitest";
+import { withScenarioHarness } from "./scenario-harness";
+import { findLatestSpan } from "./trace-selectors";
+import { captureCases } from "./attachment-capture-scenario.mjs";
+
+export function defineAttachmentCaptureTests(options: {
+  scenarioDir: string;
+  originalScenarioDir: string;
+  provider: "openai" | "anthropic" | "elevenlabs";
+  packageName: string;
+  variantKey: string;
+}) {
+  for (const mode of ["wrapped", "auto"]) {
+    test(`attachment capture policies (${options.variantKey}, ${mode})`, async () => {
+      await withScenarioHarness(async (harness) => {
+        await harness.runNodeScenarioDir({
+          scenarioDir: options.scenarioDir,
+          entry: "scenario.capture.mjs",
+          nodeArgs: mode === "auto" ? ["--import", "braintrust/hook.mjs"] : [],
+          env: {
+            CAPTURE_PACKAGE_NAME: options.packageName,
+            CAPTURE_WRAPPED: String(mode === "wrapped"),
+            BRAINTRUST_CAPTURE_ATTACHMENTS: "",
+          },
+          runContext: {
+            variantKey: options.variantKey,
+            originalScenarioDir: options.originalScenarioDir,
+            // The same Chat Completions request works on all supported SDK majors.
+            // Its real-provider recording lives in the existing v6 media cassette.
+            ...(options.provider === "openai"
+              ? { cassette: { variantKey: "openai-v6" } }
+              : {}),
+          },
+          timeoutMs: 180_000,
+        });
+        const events = harness.events();
+        for (const testCase of captureCases) {
+          const root = findLatestSpan(
+            events,
+            `attachment-capture-${testCase.name}`,
+          );
+          expect(root).toBeDefined();
+          const children = events.filter(
+            (event) =>
+              event.row.root_span_id === root?.row.root_span_id &&
+              event.span.id !== root?.span.id,
+          );
+          expect(children.length).toBeGreaterThan(0);
+          const payload = JSON.stringify(children);
+          if (testCase.enabled)
+            expect(payload).toContain("braintrust_attachment");
+          else {
+            expect(payload).not.toContain("braintrust_attachment");
+            expect(payload).not.toContain('"file_data":');
+            expect(payload).not.toContain('"b64_json":');
+            expect(payload).not.toContain('"data":');
+            expect(payload).not.toContain("data:image/png;base64,");
+          }
+          if (options.provider === "elevenlabs") {
+            expect(payload).toContain("time_to_first_token");
+            expect(payload).toContain("annotations");
+            expect(payload).not.toContain("audioBase64");
+          } else {
+            expect(payload).toContain("completion_tokens");
+            expect(
+              children.some((event) => event.row.output !== undefined),
+            ).toBe(true);
+          }
+        }
+      });
+    }, 240_000);
+  }
+}

e2e/helpers/attachment-capture-scenario.mjs

@@ -0,0 +1,149 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import {
+  initLogger,
+  wrapOpenAI,
+  wrapAnthropic,
+  wrapElevenLabs,
+} from "braintrust";
+import { MINIMAL_PNG_BASE64 } from "./media-fixtures.mjs";
+import { scopedName } from "./provider-runtime.mjs";
+
+export const captureCases = [
+  { name: "default", env: "", enabled: false },
+  { name: "local-default", env: "", global: true, local: true, enabled: false },
+  {
+    name: "local-enabled",
+    env: "false",
+    global: false,
+    local: true,
+    option: true,
+    enabled: true,
+  },
+  {
+    name: "local-disabled",
+    env: "true",
+    global: true,
+    local: true,
+    option: false,
+    enabled: false,
+  },
+  {
+    name: "local-environment",
+    env: "true",
+    global: false,
+    local: true,
+    enabled: true,
+  },
+  { name: "global-enabled", env: "false", option: true, enabled: true },
+  { name: "global-disabled", env: "true", option: false, enabled: false },
+];
+
+export async function runAttachmentCaptureScenario(provider, scenarioUrl, sdk) {
+  const wrapped = process.env.CAPTURE_WRAPPED === "true";
+  let request;
+  if (provider === "anthropic") {
+    const raw = new sdk.default({
+      apiKey: process.env.ANTHROPIC_API_KEY,
+      baseURL: process.env.ANTHROPIC_BASE_URL,
+    });
+    const client = wrapped ? wrapAnthropic(raw) : raw;
+    const data = (
+      await readFile(new URL("./test-image.png", scenarioUrl))
+    ).toString("base64");
+    request = async () => {
+      const result = await client.messages.create({
+        model: "claude-haiku-4-5",
+        max_tokens: 32,
+        temperature: 0,
+        messages: [
+          {
+            role: "user",
+            content: [
+              {
+                type: "text",
+                text: "Describe the attached image in one short sentence.",
+              },
+              {
+                type: "image",
+                source: { type: "base64", media_type: "image/png", data },
+              },
+            ],
+          },
+        ],
+      });
+      assert.ok(result.content.length > 0);
+    };
+  } else if (provider === "openai") {
+    const raw = new sdk.default({
+      apiKey: process.env.OPENAI_API_KEY,
+      baseURL: process.env.OPENAI_BASE_URL,
+    });
+    const client = wrapped ? wrapOpenAI(raw) : raw;
+    request = async () => {
+      const result = await client.chat.completions.create({
+        model: "gpt-4o-mini-2024-07-18",
+        messages: [
+          {
+            role: "user",
+            content: [
+              {
+                type: "text",
+                text: "Describe this image in three words or fewer.",
+              },
+              {
+                type: "image_url",
+                image_url: {
+                  url: `data:image/png;base64,${MINIMAL_PNG_BASE64}`,
+                },
+              },
+            ],
+          },
+        ],
+        max_tokens: 24,
+        temperature: 0,
+      });
+      assert.ok(result.choices[0].message.content);
+    };
+  } else {
+    const raw = new sdk.ElevenLabsClient({
+      apiKey: process.env.ELEVENLABS_API_KEY,
+      baseUrl: process.env.ELEVENLABS_BASE_URL,
+    });
+    const client = wrapped ? wrapElevenLabs(raw) : raw;
+    request = async () => {
+      const stream = await client.textToSpeech.streamWithTimestamps(
+        "JBFqnCBsd6RMkjVDRZzb",
+        {
+          text: "Streaming timestamps.",
+          modelId: "eleven_flash_v2_5",
+          outputFormat: "mp3_44100_128",
+        },
+      );
+      let size = 0;
+      for await (const chunk of stream) size += chunk.audioBase64.length;
+      assert.ok(size > 0);
+    };
+  }
+  const projectName = scopedName("tmp-luca-attachment-capture");
+  for (const testCase of captureCases) {
+    process.env.BRAINTRUST_CAPTURE_ATTACHMENTS = testCase.env;
+    if (testCase.local)
+      initLogger({ projectName, captureAttachments: testCase.global });
+    const logger = initLogger({
+      projectName,
+      setCurrent: !testCase.local,
+      captureAttachments: testCase.option,
+    });
+    await logger.traced(request, {
+      name: `attachment-capture-${testCase.name}`,
+      event: {
+        metadata: {
+          scenario: `${provider}-instrumentation`,
+          testRunId: process.env.BRAINTRUST_E2E_RUN_ID,
+        },
+      },
+    });
+    await logger.flush();
+  }
+}

e2e/scenarios/ai-sdk-instrumentation/scenario.test.ts

@@ -55,6 +55,7 @@ describeVariants("variants", () => {
           await runScenarioDir({
             entry: scenario.wrapperEntry,
             env: {
+              BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
               AI_SDK_PACKAGE_NAME: scenario.packageName,
               AI_SDK_OPENAI_PACKAGE_NAME: scenario.openaiModuleName,
               AI_SDK_SUPPORTS_EVALUATE_STRING_MODEL: String(
@@ -116,6 +117,7 @@ describeVariants("variants", () => {
           await runNodeScenarioDir({
             entry: scenario.autoEntry,
             env: {
+              BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
               AI_SDK_PACKAGE_NAME: scenario.packageName,
               AI_SDK_OPENAI_PACKAGE_NAME: scenario.openaiModuleName,
               AI_SDK_SUPPORTS_EVALUATE_STRING_MODEL: String(

e2e/scenarios/anthropic-instrumentation/scenario.capture.mjs

@@ -0,0 +1,10 @@
+import { runMain } from "../../helpers/provider-runtime.mjs";
+import { runAttachmentCaptureScenario } from "../../helpers/attachment-capture-scenario.mjs";
+
+runMain(async () =>
+  runAttachmentCaptureScenario(
+    "anthropic",
+    import.meta.url,
+    await import(process.env.CAPTURE_PACKAGE_NAME),
+  ),
+);

e2e/scenarios/anthropic-instrumentation/scenario.test.ts

@@ -1,3 +1,4 @@
+import { defineAttachmentCaptureTests } from "../../helpers/attachment-capture-assertions";
 import { describe } from "vitest";
 import {
   prepareScenarioDir,
@@ -52,7 +53,10 @@ describe.concurrent("variants", () => {
         runScenario: async ({ runScenarioDir }) => {
           await runScenarioDir({
             entry: scenario.wrapperEntry,
-            env: { ANTHROPIC_PACKAGE_NAME: scenario.dependencyName },
+            env: {
+              BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
+              ANTHROPIC_PACKAGE_NAME: scenario.dependencyName,
+            },
             runContext: {
               variantKey: scenario.snapshotName,
               originalScenarioDir,
@@ -78,7 +82,10 @@ describe.concurrent("variants", () => {
         runScenario: async ({ runNodeScenarioDir }) => {
           await runNodeScenarioDir({
             entry: scenario.autoEntry,
-            env: { ANTHROPIC_PACKAGE_NAME: scenario.dependencyName },
+            env: {
+              BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
+              ANTHROPIC_PACKAGE_NAME: scenario.dependencyName,
+            },
             nodeArgs: ["--import", "braintrust/hook.mjs"],
             runContext: {
               variantKey: scenario.snapshotName,
@@ -100,3 +107,13 @@ describe.concurrent("variants", () => {
     });
   }
 });
+
+for (const scenario of anthropicScenarios) {
+  defineAttachmentCaptureTests({
+    scenarioDir,
+    originalScenarioDir,
+    provider: "anthropic",
+    packageName: scenario.dependencyName,
+    variantKey: scenario.snapshotName,
+  });
+}

e2e/scenarios/elevenlabs-instrumentation/scenario.capture.mjs

@@ -0,0 +1,10 @@
+import { runMain } from "../../helpers/provider-runtime.mjs";
+import { runAttachmentCaptureScenario } from "../../helpers/attachment-capture-scenario.mjs";
+
+runMain(async () =>
+  runAttachmentCaptureScenario(
+    "elevenlabs",
+    import.meta.url,
+    await import(process.env.CAPTURE_PACKAGE_NAME),
+  ),
+);

e2e/scenarios/elevenlabs-instrumentation/scenario.test.ts

@@ -1,3 +1,4 @@
+import { defineAttachmentCaptureTests } from "../../helpers/attachment-capture-assertions";
 import { describe, expect, test } from "vitest";
 import {
   prepareScenarioDir,
@@ -30,6 +31,7 @@ for (const variant of ["elevenlabs-v2", "elevenlabs-v2-latest"]) {
                 ? ["--import", "braintrust/hook.mjs"]
                 : [],
               env: {
+                BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
                 ELEVENLABS_PACKAGE_NAME: variant.replace(
                   "elevenlabs-",
                   "elevenlabs-sdk-",
@@ -143,3 +145,13 @@ for (const variant of ["elevenlabs-v2", "elevenlabs-v2-latest"]) {
     }
   });
 }
+
+for (const variant of ["elevenlabs-v2", "elevenlabs-v2-latest"]) {
+  defineAttachmentCaptureTests({
+    scenarioDir,
+    originalScenarioDir,
+    provider: "elevenlabs",
+    packageName: variant.replace("elevenlabs-", "elevenlabs-sdk-"),
+    variantKey: variant,
+  });
+}

e2e/scenarios/google-genai-instrumentation/scenario.test.ts

@@ -56,7 +56,10 @@ describe.concurrent("variants", () => {
           runScenario: async ({ runScenarioDir }) => {
             await runScenarioDir({
               entry: scenario.wrapperEntry,
-              env: { GOOGLE_GENAI_PACKAGE_NAME: scenario.dependencyName },
+              env: {
+                BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
+                GOOGLE_GENAI_PACKAGE_NAME: scenario.dependencyName,
+              },
               runContext: {
                 variantKey: scenario.snapshotName,
                 originalScenarioDir,
@@ -75,7 +78,10 @@ describe.concurrent("variants", () => {
           runScenario: async ({ runNodeScenarioDir }) => {
             await runNodeScenarioDir({
               entry: scenario.autoEntry,
-              env: { GOOGLE_GENAI_PACKAGE_NAME: scenario.dependencyName },
+              env: {
+                BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
+                GOOGLE_GENAI_PACKAGE_NAME: scenario.dependencyName,
+              },
               nodeArgs: ["--import", "braintrust/hook.mjs"],
               runContext: {
                 variantKey: scenario.snapshotName,

e2e/scenarios/google-generative-ai-instrumentation/scenario.test.ts

@@ -30,7 +30,10 @@ describe.concurrent("variants", () => {
         runScenario: async ({ runScenarioDir }) => {
           await runScenarioDir({
             entry: "scenario.ts",
-            env: { GOOGLE_GENERATIVE_AI_PACKAGE_NAME: scenario.dependencyName },
+            env: {
+              BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
+              GOOGLE_GENERATIVE_AI_PACKAGE_NAME: scenario.dependencyName,
+            },
             runContext: {
               originalScenarioDir,
               variantKey: scenario.snapshotName,
@@ -50,7 +53,10 @@ describe.concurrent("variants", () => {
           await runNodeScenarioDir({
             entry: "scenario.mjs",
             nodeArgs: ["--import", "braintrust/hook.mjs"],
-            env: { GOOGLE_GENERATIVE_AI_PACKAGE_NAME: scenario.dependencyName },
+            env: {
+              BRAINTRUST_CAPTURE_ATTACHMENTS: "true",
+              GOOGLE_GENERATIVE_AI_PACKAGE_NAME: scenario.dependencyName,
+            },
             runContext: {
               originalScenarioDir,
               variantKey: scenario.snapshotName,