From 40351df9d659a9b6eb1608c6d92ff4bc9fe799a4 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Fri, 28 Aug 2026 12:08:29 -0600 Subject: [PATCH 1/3] use new activity source exception handling --- src/Braintrust.Sdk.Anthropic/InstrumentedMessageService.cs | 5 ++--- src/Braintrust.Sdk.OpenAI/InstrumentedChatClient.cs | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Braintrust.Sdk.Anthropic/InstrumentedMessageService.cs b/src/Braintrust.Sdk.Anthropic/InstrumentedMessageService.cs index cb6859c..f4496a2 100644 --- a/src/Braintrust.Sdk.Anthropic/InstrumentedMessageService.cs +++ b/src/Braintrust.Sdk.Anthropic/InstrumentedMessageService.cs @@ -5,7 +5,6 @@ using Anthropic.Models.Messages; using Anthropic.Services; using Anthropic.Services.Messages; -using OpenTelemetry.Trace; namespace Braintrust.Sdk.Anthropic; @@ -62,7 +61,7 @@ public async Task Create( if (activity != null) { activity.SetStatus(ActivityStatusCode.Error, ex.Message); - activity.RecordException(ex); + activity.AddException(ex); } throw; } @@ -115,7 +114,7 @@ public async IAsyncEnumerable CreateStreaming( if (activity != null) { activity.SetStatus(ActivityStatusCode.Error, ex.Message); - activity.RecordException(ex); + activity.AddException(ex); } throw; } diff --git a/src/Braintrust.Sdk.OpenAI/InstrumentedChatClient.cs b/src/Braintrust.Sdk.OpenAI/InstrumentedChatClient.cs index a78cdff..69a4ffc 100644 --- a/src/Braintrust.Sdk.OpenAI/InstrumentedChatClient.cs +++ b/src/Braintrust.Sdk.OpenAI/InstrumentedChatClient.cs @@ -3,7 +3,6 @@ using System.Text.Json; using System.Text.Json.Nodes; using OpenAI.Chat; -using OpenTelemetry.Trace; namespace Braintrust.Sdk.OpenAI; @@ -71,7 +70,7 @@ public override ClientResult CompleteChat(IEnumerable> CompleteChatAsync(IEnum if (activity != null) { activity.SetStatus(ActivityStatusCode.Error, ex.Message); - activity.RecordException(ex); + activity.AddException(ex); } // intentionally re-throwing original exception throw; From fe5eafac2db0da843aae43b7fa9ee3e0c63e1a9a Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Wed, 26 Aug 2026 11:35:23 -0600 Subject: [PATCH 2/3] fix uri printing for experiments with whitespace --- src/Braintrust.Sdk/Eval/Eval.cs | 2 +- tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs | 28 +++++++++++++++++++ .../Eval/MockBraintrustApiClient.cs | 10 +++++-- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/Braintrust.Sdk/Eval/Eval.cs b/src/Braintrust.Sdk/Eval/Eval.cs index 461c8a5..d57843c 100644 --- a/src/Braintrust.Sdk/Eval/Eval.cs +++ b/src/Braintrust.Sdk/Eval/Eval.cs @@ -586,7 +586,7 @@ private static string CreateExperimentUrl( "experiments", Uri.EscapeDataString(experimentName)); - return new UriBuilder(baseUri.Scheme, baseUri.Host, baseUri.Port, "/" + path).Uri.ToString(); + return new UriBuilder(baseUri.Scheme, baseUri.Host, baseUri.Port, "/" + path).Uri.AbsoluteUri; } diff --git a/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs b/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs index c04bb52..cdea0e2 100644 --- a/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs +++ b/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs @@ -986,6 +986,34 @@ public async Task ScoreForTaskException_Override_ReturnsCustomFallbackScore() Assert.Equal(0.75, GetScore(scoreSpan, "custom_scorer")); } + [Fact] + public async Task ExperimentUrlEscapesNamesWithSpaces() + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "my project") + ); + + var mockClient = new MockBraintrustApiClient(orgName: "Braintrust SDKs", projectName: "my project"); + + var eval = await Eval.NewBuilder() + .Name("my eval") + .Config(config) + .ApiClient(mockClient) + .BtqlClient(new MockBtqlClient()) + .Cases(new DatasetCase("strawberry", "fruit")) + .TaskFunction(food => "fruit") + .Scorers(new FunctionScorer("exact", (expected, actual) => expected == actual ? 1.0 : 0.0)) + .BuildAsync(); + + var result = await eval.RunAsync(); + + Assert.Equal( + "https://braintrust.dev/app/Braintrust%20SDKs/p/my%20project/experiments/my%20eval", + result.ExperimentUrl); + } + // ------------------------------------------------------------------------- // Test helper methods // ------------------------------------------------------------------------- diff --git a/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs b/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs index 0796e49..554b09b 100644 --- a/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs +++ b/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs @@ -7,8 +7,14 @@ namespace Braintrust.Sdk.Tests.Eval; /// internal class MockBraintrustApiClient : IBraintrustApiClient { - private readonly OrganizationInfo _orgInfo = new OrganizationInfo("test-org-id", "test-org"); - private readonly Project _project = new Project("test-project-id", "test-project", "test-org-id"); + private readonly OrganizationInfo _orgInfo; + private readonly Project _project; + + public MockBraintrustApiClient(string orgName = "test-org", string projectName = "test-project") + { + _orgInfo = new OrganizationInfo("test-org-id", orgName); + _project = new Project("test-project-id", projectName, "test-org-id"); + } /// /// The last CreateExperimentRequest received by GetOrCreateExperiment. From d5ff94836e80e69b12aecc149034f98aada14653 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Wed, 26 Aug 2026 11:44:13 -0600 Subject: [PATCH 3/3] support remote datasets --- README.md | 53 +- docs/api-client.md | 19 +- .../AgentFrameworkInstrumentation/Program.cs | 3 +- examples/AnthropicInstrumentation/Program.cs | 3 +- examples/ApiClientExample/Program.cs | 4 +- .../AzureOpenAIInstrumentation/Program.cs | 3 +- examples/ClassifiersExample/Program.cs | 2 +- examples/EvalExample/Program.cs | 4 +- examples/OpenAIInstrumentation/Program.cs | 3 +- examples/SimpleOpenTelemetry/Program.cs | 3 +- examples/TraceScoring/Program.cs | 2 +- .../SerializerSettings.cs | 4 +- src/Braintrust.Sdk/Api/BraintrustApiClient.cs | 238 ++------ .../Api/BraintrustOpenApiClient.cs | 190 +++++++ .../Api/DefaultBraintrustApiClient.cs | 182 ++---- .../Api/IBraintrustApiClient.cs | 3 +- src/Braintrust.Sdk/Api/Internal/BtqlClient.cs | 114 ++-- .../Api/Internal/IBtqlClient.cs | 9 - src/Braintrust.Sdk/Api/Models.cs | 39 +- src/Braintrust.Sdk/Braintrust.cs | 90 ++- .../Eval/DatasetBrainstoreImpl.cs | 231 ++++++++ src/Braintrust.Sdk/Eval/DatasetCase.cs | 6 + .../Eval/DatasetInMemoryImpl.cs | 2 +- src/Braintrust.Sdk/Eval/Eval.cs | 373 +++++++++--- src/Braintrust.Sdk/Eval/IDataset.cs | 181 +++++- src/Braintrust.Sdk/Origin.cs | 21 + .../Api/BraintrustApiClientTest.cs | 213 ------- .../Api/BraintrustOpenApiClientTest.cs | 142 +++++ .../Api/DefaultBraintrustApiClientTest.cs | 327 ----------- .../Api/DeprecatedApisTest.cs | 80 +++ .../Api/Internal/BtqlClientTest.cs | 80 +-- .../Braintrust.Sdk.Tests.csproj | 2 +- tests/Braintrust.Sdk.Tests/BraintrustTest.cs | 139 +++++ .../Braintrust.Sdk.Tests/Eval/BtqlTestData.cs | 24 + .../Eval/ClassifierTest.cs | 23 +- .../Eval/DatasetBrainstoreImplTest.cs | 514 +++++++++++++++++ tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs | 530 +++++++++++++++--- .../Eval/EvalTraceTest.cs | 2 +- .../Eval/MockBraintrustApiClient.cs | 55 -- .../Eval/MockBtqlClient.cs | 50 -- .../Eval/StubBraintrustApi.cs | 218 +++++++ .../Braintrust.Sdk.Tests/QueuedHttpHandler.cs | 40 ++ 42 files changed, 2895 insertions(+), 1326 deletions(-) create mode 100644 src/Braintrust.Sdk/Api/BraintrustOpenApiClient.cs delete mode 100644 src/Braintrust.Sdk/Api/Internal/IBtqlClient.cs create mode 100644 src/Braintrust.Sdk/Eval/DatasetBrainstoreImpl.cs create mode 100644 src/Braintrust.Sdk/Origin.cs delete mode 100644 tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs create mode 100644 tests/Braintrust.Sdk.Tests/Api/BraintrustOpenApiClientTest.cs delete mode 100644 tests/Braintrust.Sdk.Tests/Api/DefaultBraintrustApiClientTest.cs create mode 100644 tests/Braintrust.Sdk.Tests/Api/DeprecatedApisTest.cs create mode 100644 tests/Braintrust.Sdk.Tests/Eval/BtqlTestData.cs create mode 100644 tests/Braintrust.Sdk.Tests/Eval/DatasetBrainstoreImplTest.cs delete mode 100644 tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs delete mode 100644 tests/Braintrust.Sdk.Tests/Eval/MockBtqlClient.cs create mode 100644 tests/Braintrust.Sdk.Tests/Eval/StubBraintrustApi.cs create mode 100644 tests/Braintrust.Sdk.Tests/QueuedHttpHandler.cs diff --git a/README.md b/README.md index a1d265c..5e79e1d 100644 --- a/README.md +++ b/README.md @@ -54,13 +54,64 @@ dotnet add package Braintrust.Sdk.AgentFramework ``` +## Datasets + +Evals can read their cases straight from a Braintrust dataset instead of being written out by +hand: + +```csharp +using Braintrust.Sdk.Eval; + +var braintrust = Braintrust.Get(); + +// Reads from the configured default project. Resolves the dataset id now; rows are fetched a +// page at a time as the eval reads them. +var dataset = await braintrust.FetchDatasetAsync("my-dataset"); + +using var eval = await braintrust.EvalBuilder() + .Name("my-eval") + .Dataset(dataset) + .TaskFunction(input => Classify(input)) + .Scorers(new FunctionScorer("exact", (expected, actual) => expected == actual ? 1.0 : 0.0)) + .BuildAsync(); + +await eval.RunAsync(); +``` + +Disposing the eval closes the HTTP client it opened for itself. An API client you pass to the +builder yourself is left alone - it stays yours to dispose. + +Each case's `input` and `expected` are deserialized into the type arguments you pick, so +`FetchDatasetAsync("my-dataset")` gives you typed cases. + +`expected` is optional in a dataset, and the type arguments are not nullable, so rows the default +deserializer cannot handle need an `inputConverter`/`expectedConverter` to say what a missing or +oddly-shaped field means: + +```csharp +var dataset = await braintrust.FetchDatasetAsync( + "my-dataset", + expectedConverter: e => e.ValueKind == JsonValueKind.Null ? "" : e.GetString()!); +``` + +Passing `version` pins the read to a transaction id. Leaving it null - the default - resolves the +latest version when enumeration starts and reads every page as of that one version, so a dataset +written to mid-run still produces a consistent eval. Either way the experiment records which +dataset and version it ran against, and each eval row links back to the dataset record it came +from. + +For a dataset outside the configured project, or one you already have the id of, use +`Dataset.FetchFromBraintrustAsync<...>(apiClient, projectName, datasetName)` and +`Dataset.FromId<...>(apiClient, datasetId)`. These factories require a caller-owned API client and +never dispose it. + ## Low-level API client Beyond evals and tracing, the SDK ships a client for the full [Braintrust REST API](https://api.braintrust.dev), generated from Braintrust's public OpenAPI spec: ```csharp -using var client = DefaultBraintrustApiClient.Of(BraintrustConfig.FromEnvironment()); +using var client = BraintrustOpenApiClient.Of(BraintrustConfig.FromEnvironment()); var api = client.Api; // every Braintrust REST endpoint ``` diff --git a/docs/api-client.md b/docs/api-client.md index c431d6d..fcc35d5 100644 --- a/docs/api-client.md +++ b/docs/api-client.md @@ -21,7 +21,7 @@ because `Braintrust.Sdk` references it with `PrivateAssets="all"`. See ## Basic usage -`DefaultBraintrustApiClient` is the SDK's own client; its `Api` property is the generated +`BraintrustOpenApiClient` is the SDK's own client; its `Api` property is the generated client, already wired up with the base URL, bearer auth and timeout from your config. ```csharp @@ -29,7 +29,7 @@ using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; using Generated = Braintrust.Sdk.Api.Generated; -using var client = DefaultBraintrustApiClient.Of(BraintrustConfig.FromEnvironment()); +using var client = BraintrustOpenApiClient.Of(BraintrustConfig.FromEnvironment()); Generated.IBraintrustGeneratedApiClient api = client.Api; // Create a project (POST /v1/project upserts by name). @@ -42,7 +42,7 @@ Generated.Project project = await api.PostProjectAsync(new Generated.CreateProje Console.WriteLine($"{project.Id} {project.Name}"); ``` -The generated client borrows the `HttpClient` owned by `DefaultBraintrustApiClient`, so keep +The generated client borrows the `HttpClient` owned by `BraintrustOpenApiClient`, so keep that instance alive for as long as you use `Api`. ### Runnable example @@ -165,9 +165,9 @@ serialize that field yourself if you need to. ### Errors -Generated calls throw `Generated.ApiException` (or `Generated.ApiException`), not the -SDK's `Braintrust.Sdk.Api.ApiException` - only the SDK's own wrapper methods translate. The -server's message is in `ex.Response`: +Calls handled by the generated OpenAPI client throw `Generated.ApiException` (or +`Generated.ApiException`), including calls made by the SDK's wrapper methods. The server's +message is in `ex.Response`: ```csharp try @@ -191,14 +191,13 @@ The spec is generated from the API's own types, but a few declared filters are r runtime - `project_automation_name` on `GET /v1/project_automation`, for instance, comes back `400 Extraneous key`. Filter client-side when that happens. -`/api/apikey/login` is not in the spec at all. `DefaultBraintrustApiClient` issues it by hand -and exposes the result through `GetProjectAndOrgInfo`. +`POST /btql` is absent from the spec, so `BraintrustOpenApiClient` implements that operation +itself. Project and organization lookup use the generated endpoints. ## Compatibility The generated surface tracks whatever spec ref the build pinned, so it is **not** covered by -the SDK's own compatibility promises: bumping the ref can rename a class or change a -signature. Prefer `IBraintrustApiClient`'s methods where they already cover what you need. +the SDK's own compatibility promises: bumping the ref can rename a class or change a signature. ## Bumping the spec diff --git a/examples/AgentFrameworkInstrumentation/Program.cs b/examples/AgentFrameworkInstrumentation/Program.cs index 3fb7aad..fe398a2 100644 --- a/examples/AgentFrameworkInstrumentation/Program.cs +++ b/examples/AgentFrameworkInstrumentation/Program.cs @@ -62,7 +62,8 @@ static async Task Main(string[] args) Console.WriteLine($"Messages: {response.Messages.Count}"); // Print Braintrust link - var url = await braintrust.GetProjectUriAsync() + var projectUri = await braintrust.GetProjectUriAsync(); + var url = projectUri.AbsoluteUri + $"/logs?r={rootActivity.TraceId}&s={rootActivity.SpanId}"; Console.WriteLine($"\n View your trace in Braintrust: {url}\n"); } diff --git a/examples/AnthropicInstrumentation/Program.cs b/examples/AnthropicInstrumentation/Program.cs index 61f9cc2..f57b8ea 100644 --- a/examples/AnthropicInstrumentation/Program.cs +++ b/examples/AnthropicInstrumentation/Program.cs @@ -32,7 +32,8 @@ static async Task Main() { await MessageCompletionExample(instrumentedClient); await MessageStreamingExample(instrumentedClient); - var url = await braintrust.GetProjectUriAsync() + var projectUri = await braintrust.GetProjectUriAsync(); + var url = projectUri.AbsoluteUri + $"/logs?r={rootActivity.TraceId}&s={rootActivity.SpanId}"; Console.WriteLine($"\n\n Example complete! View your data in Braintrust: {url}\n"); } diff --git a/examples/ApiClientExample/Program.cs b/examples/ApiClientExample/Program.cs index 8c47d07..99c9377 100644 --- a/examples/ApiClientExample/Program.cs +++ b/examples/ApiClientExample/Program.cs @@ -32,10 +32,10 @@ class Program static async Task Main(string[] args) { - // DefaultBraintrustApiClient is the SDK's own client; its Api property is the + // BraintrustOpenApiClient is the SDK's own client; its Api property is the // generated client, already wired up with the base URL, bearer auth and timeout // from the config. - using var client = DefaultBraintrustApiClient.Of(BraintrustConfig.FromEnvironment()); + using var client = BraintrustOpenApiClient.Of(BraintrustConfig.FromEnvironment()); Generated.IBraintrustGeneratedApiClient api = client.Api; // Pick the project to read from, and resolve its org for the banner. diff --git a/examples/AzureOpenAIInstrumentation/Program.cs b/examples/AzureOpenAIInstrumentation/Program.cs index b6c2435..6754f8a 100644 --- a/examples/AzureOpenAIInstrumentation/Program.cs +++ b/examples/AzureOpenAIInstrumentation/Program.cs @@ -56,7 +56,8 @@ static async Task Main(string[] args) if (rootActivity != null) { await ChatCompletionsExample(instrumentedClient, deploymentName); - var url = await braintrust.GetProjectUriAsync() + var projectUri = await braintrust.GetProjectUriAsync(); + var url = projectUri.AbsoluteUri + $"/logs?r={rootActivity.TraceId}&s={rootActivity.SpanId}"; Console.WriteLine($"\n\n Example complete! View your data in Braintrust: {url}\n"); } diff --git a/examples/ClassifiersExample/Program.cs b/examples/ClassifiersExample/Program.cs index 4727ddd..66097e0 100644 --- a/examples/ClassifiersExample/Program.cs +++ b/examples/ClassifiersExample/Program.cs @@ -137,7 +137,7 @@ static async Task Main() .Select(m => DatasetCase.Of(m.Input, m.Expected)) .ToArray(); - var eval = await braintrust + using var eval = await braintrust .EvalBuilder() .Name($"dotnet-classifiers-example-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}") .Tags("classifiers-example", "dotnet-sdk") diff --git a/examples/EvalExample/Program.cs b/examples/EvalExample/Program.cs index 0640b32..ee6912c 100644 --- a/examples/EvalExample/Program.cs +++ b/examples/EvalExample/Program.cs @@ -65,7 +65,7 @@ async Task GetFoodType(string food) } // Create and run the evaluation - var eval = await braintrust + using var eval = await braintrust .EvalBuilder() .Name($"dotnet-eval-x-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}") // Experiment-level tags and metadata (shown in the Braintrust UI for the experiment) @@ -75,6 +75,8 @@ async Task GetFoodType(string food) { "model", "gpt-4o-mini" }, { "description", "Classifies food items as fruit or vegetable" } }) + // instead of Cases, you can use a dataset from Braintrust + // .Dataset(await braintrust.FetchDatasetAsync("food")) .Cases( DatasetCase.Of("strawberry", "fruit"), DatasetCase.Of("asparagus", "vegetable"), diff --git a/examples/OpenAIInstrumentation/Program.cs b/examples/OpenAIInstrumentation/Program.cs index 4ea6716..95ebfae 100644 --- a/examples/OpenAIInstrumentation/Program.cs +++ b/examples/OpenAIInstrumentation/Program.cs @@ -31,7 +31,8 @@ static async Task Main(string[] args) if (rootActivity != null) { await ChatCompletionsExample(instrumentedClient); - var url = await braintrust.GetProjectUriAsync() + var projectUri = await braintrust.GetProjectUriAsync(); + var url = projectUri.AbsoluteUri + $"/logs?r={rootActivity.TraceId}&s={rootActivity.SpanId}"; Console.WriteLine($"\n\n Example complete! View your data in Braintrust: {url}\n"); } diff --git a/examples/SimpleOpenTelemetry/Program.cs b/examples/SimpleOpenTelemetry/Program.cs index d968db0..ce9ea6a 100644 --- a/examples/SimpleOpenTelemetry/Program.cs +++ b/examples/SimpleOpenTelemetry/Program.cs @@ -13,7 +13,8 @@ static async Task Main(string[] args) ArgumentNullException.ThrowIfNull(activity); Console.WriteLine("Performing simple operation..."); activity.SetTag("some boolean attribute", true); - url = await braintrust.GetProjectUriAsync() + $"/logs?r={activity.TraceId}&s={activity.SpanId}"; + var projectUri = await braintrust.GetProjectUriAsync(); + url = projectUri.AbsoluteUri + $"/logs?r={activity.TraceId}&s={activity.SpanId}"; } Console.WriteLine($"\n\n Example complete! View your data in Braintrust: {url}"); } diff --git a/examples/TraceScoring/Program.cs b/examples/TraceScoring/Program.cs index e49e0a5..a13d365 100644 --- a/examples/TraceScoring/Program.cs +++ b/examples/TraceScoring/Program.cs @@ -65,7 +65,7 @@ Dictionary CountFruits(string fruitList) // Scorer: uses the trace to verify each LLM call returned a numeric string var traceScorer = new FruitTraceScorer(); - var eval = await braintrust + using var eval = await braintrust .EvalBuilder>() .Name($"trace-scoring-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}") .Tags("trace-scoring", "dotnet-sdk", "multi-call") diff --git a/src/Braintrust.Sdk.Api.Generated/SerializerSettings.cs b/src/Braintrust.Sdk.Api.Generated/SerializerSettings.cs index 09fbf3d..a1d53c1 100644 --- a/src/Braintrust.Sdk.Api.Generated/SerializerSettings.cs +++ b/src/Braintrust.Sdk.Api.Generated/SerializerSettings.cs @@ -14,8 +14,8 @@ static partial void UpdateJsonSerializerSettings(JsonSerializerOptions settings) { // The generator writes every unset optional member as an explicit null, and the API // validates nullability strictly: creating an automation whose action never mentions - // formatting_prompt was rejected with "Expected string, received null". Omitting - // nulls is also what the SDK's hand-rolled client did. + // formatting_prompt was rejected with "Expected string, received null", so unset + // members are omitted instead. // // The trade-off is that an explicit null can no longer be sent to clear a field on // PATCH. Braintrust's PATCH endpoints treat an absent field as "leave alone" and diff --git a/src/Braintrust.Sdk/Api/BraintrustApiClient.cs b/src/Braintrust.Sdk/Api/BraintrustApiClient.cs index 2369d74..6a4fb31 100644 --- a/src/Braintrust.Sdk/Api/BraintrustApiClient.cs +++ b/src/Braintrust.Sdk/Api/BraintrustApiClient.cs @@ -1,228 +1,76 @@ -using System.Net.Http.Headers; -using System.Net.Http.Json; -using System.Text.Json; -using System.Text.Json.Serialization; using Braintrust.Sdk.Config; namespace Braintrust.Sdk.Api; /// -/// Hand-rolled implementation of the Braintrust API client. +/// Use instead. /// -/// -/// Superseded by , which is backed by the client -/// generated from the Braintrust OpenAPI spec and so stays in step with the API rather -/// than needing each endpoint written out by hand. This type is kept only so existing -/// callers keep compiling; it receives no new endpoints and will be removed. -/// -[Obsolete("Use DefaultBraintrustApiClient, which is backed by the generated OpenAPI client. " + - "This hand-rolled client is no longer maintained and will be removed in a future release.")] +[Obsolete("Use BraintrustOpenApiClient instead.")] public class BraintrustApiClient : IBraintrustApiClient, IDisposable { - private readonly BraintrustConfig _config; - private readonly HttpClient _httpClient; - private readonly bool _ownsHttpClient; - private readonly JsonSerializerOptions _jsonOptions; + private readonly DefaultBraintrustApiClient _client; - public static BraintrustApiClient Of(BraintrustConfig config) - { - return new BraintrustApiClient(config); - } + public static BraintrustApiClient Of(BraintrustConfig config) => new(config); internal BraintrustApiClient(BraintrustConfig config, HttpClient? httpClient = null) { - _config = config ?? throw new ArgumentNullException(nameof(config)); - - if (httpClient == null) - { - _httpClient = CreateDefaultHttpClient(config); - _ownsHttpClient = true; - } - else - { - _httpClient = httpClient; - _ownsHttpClient = false; - } - - _jsonOptions = CreateJsonOptions(); + var openApiClient = httpClient is null + ? new BraintrustOpenApiClient(config) + : new BraintrustOpenApiClient(config, new HttpClientHandlerAdapter(httpClient)); + _client = new DefaultBraintrustApiClient(config, openApiClient, ownsClient: true); } - private static HttpClient CreateDefaultHttpClient(BraintrustConfig config) - { - return new HttpClient - { - BaseAddress = new Uri(config.ApiUrl), - Timeout = config.RequestTimeout - }; - } + public Task GetOrCreateProject(string projectName) + => _client.GetOrCreateProject(projectName); - private static JsonSerializerOptions CreateJsonOptions() - { - return new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - PropertyNameCaseInsensitive = true - }; - } + public Task GetProject(string projectId) + => _client.GetProject(projectId); - public async Task GetOrCreateProject(string projectName) - { - var request = new CreateProjectRequest(projectName); - return await PostAsync("/v1/project", request).ConfigureAwait(false); - } + public Task GetOrCreateExperiment(CreateExperimentRequest request) + => _client.GetOrCreateExperiment(request); - public async Task GetProject(string projectId) - { - try - { - return await GetAsync($"/v1/project/{projectId}").ConfigureAwait(false); - } - catch (ApiException ex) when (ex.StatusCode == 404) - { - return null; - } - } + public Task GetProjectAndOrgInfo() + => _client.GetProjectAndOrgInfo(); - public async Task GetOrCreateExperiment(CreateExperimentRequest request) - { - return await PostAsync("/v1/experiment", request) - .ConfigureAwait(false); - } - - public async Task GetProjectAndOrgInfo() - { - if (_config.DefaultProjectId != null) - { - return await GetProjectAndOrgInfo(_config.DefaultProjectId).ConfigureAwait(false); - } + public Task GetProjectAndOrgInfo(string projectId) + => _client.GetProjectAndOrgInfo(projectId); - if (_config.DefaultProjectName != null) - { - var project = await GetOrCreateProject(_config.DefaultProjectName).ConfigureAwait(false); - return await GetProjectAndOrgInfo(project.Id).ConfigureAwait(false); - } + public Task GetOrCreateProjectAndOrgInfo() + => _client.GetOrCreateProjectAndOrgInfo(); - return null; - } + public void Dispose() => _client.Dispose(); - public async Task GetProjectAndOrgInfo(string projectId) + /// + /// Forwards requests through an injected without taking ownership. + /// + private sealed class HttpClientHandlerAdapter(HttpClient httpClient) : HttpMessageHandler { - var project = await GetProject(projectId).ConfigureAwait(false); - if (project == null) + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) { - return null; - } - - var loginResponse = await Login().ConfigureAwait(false); - var orgInfo = loginResponse.OrgInfo.FirstOrDefault(org => - string.Equals(org.Id, project.OrgId, StringComparison.OrdinalIgnoreCase)); - - if (orgInfo == null) - { - throw new ApiException($"Organization {project.OrgId} not found for project {projectId}"); - } - - return new OrganizationAndProjectInfo(orgInfo, project); - } - - public async Task GetOrCreateProjectAndOrgInfo() - { - Project project; + using var forwarded = new HttpRequestMessage(request.Method, request.RequestUri) + { + Version = request.Version, + VersionPolicy = request.VersionPolicy, + }; - if (_config.DefaultProjectId != null) - { - var existingProject = await GetProject(_config.DefaultProjectId).ConfigureAwait(false); - if (existingProject == null) + foreach (var header in request.Headers) { - throw new ApiException($"Project with ID {_config.DefaultProjectId} not found"); + forwarded.Headers.TryAddWithoutValidation(header.Key, header.Value); } - project = existingProject; - } - else if (_config.DefaultProjectName != null) - { - project = await GetOrCreateProject(_config.DefaultProjectName).ConfigureAwait(false); - } - else - { - throw new InvalidOperationException("Either DefaultProjectId or DefaultProjectName must be set in config"); - } - - var loginResponse = await Login().ConfigureAwait(false); - var orgInfo = loginResponse.OrgInfo.FirstOrDefault(org => - string.Equals(org.Id, project.OrgId, StringComparison.OrdinalIgnoreCase)); - - if (orgInfo == null) - { - throw new ApiException($"Organization {project.OrgId} not found"); - } - - return new OrganizationAndProjectInfo(orgInfo, project); - } - - private async Task Login() - { - var apiKey = await _config.GetRequiredApiKeyAsync().ConfigureAwait(false); - var request = new LoginRequest(apiKey); - return await PostAsync("/api/apikey/login", request, apiKey: apiKey) - .ConfigureAwait(false); - } - private async Task GetAsync(string path, CancellationToken cancellationToken = default) - { - var apiKey = await _config.GetRequiredApiKeyAsync(cancellationToken).ConfigureAwait(false); - using var request = new HttpRequestMessage(HttpMethod.Get, path); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey); - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - - using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); - return await HandleResponseAsync(response, cancellationToken).ConfigureAwait(false); - } - - private async Task PostAsync( - string path, - TRequest body, - CancellationToken cancellationToken = default, - string? apiKey = null) - { - apiKey ??= await _config.GetRequiredApiKeyAsync(cancellationToken).ConfigureAwait(false); - using var request = new HttpRequestMessage(HttpMethod.Post, path); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey); - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - request.Content = JsonContent.Create(body, options: _jsonOptions); - - using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); - return await HandleResponseAsync(response, cancellationToken).ConfigureAwait(false); - } - - private async Task HandleResponseAsync(HttpResponseMessage response, CancellationToken cancellationToken) - { - if (response.IsSuccessStatusCode) - { - var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - var result = JsonSerializer.Deserialize(content, _jsonOptions); - - if (result == null) + if (request.Content is not null) { - throw new ApiException("Failed to deserialize API response"); + forwarded.Content = new ByteArrayContent( + await request.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false)); + foreach (var header in request.Content.Headers) + { + forwarded.Content.Headers.TryAddWithoutValidation(header.Key, header.Value); + } } - return result; - } - else - { - var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - throw new ApiException( - (int)response.StatusCode, - $"API request failed with status {(int)response.StatusCode}: {content}"); - } - } - - public void Dispose() - { - if (_ownsHttpClient) - { - _httpClient.Dispose(); + return await httpClient.SendAsync(forwarded, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Braintrust.Sdk/Api/BraintrustOpenApiClient.cs b/src/Braintrust.Sdk/Api/BraintrustOpenApiClient.cs new file mode 100644 index 0000000..1589b0a --- /dev/null +++ b/src/Braintrust.Sdk/Api/BraintrustOpenApiClient.cs @@ -0,0 +1,190 @@ +using System.Net; +using System.Text.Json; +using Braintrust.Sdk.Api.Internal; +using Braintrust.Sdk.Config; +using Generated = Braintrust.Sdk.Api.Generated; + +namespace Braintrust.Sdk.Api; + +/// +/// OpenAPI-backed client used by the Braintrust SDK. It owns the configured HTTP transport, +/// exposes the generated client through , and owns the SDK's project, app-link, +/// and BTQL operations. +/// +/// Use for REST operations. +/// +public sealed class BraintrustOpenApiClient : IDisposable +{ + private readonly BraintrustConfig _config; + private readonly HttpClient _httpClient; + private readonly Generated.IBraintrustGeneratedApiClient _api; + private readonly BtqlClient _btqlClient; + + public static BraintrustOpenApiClient Of(BraintrustConfig config) => new(config); + + public BraintrustOpenApiClient(BraintrustConfig config) + : this(config, innerHandler: null) + { + } + + /// + /// Transport to send through, wrapped so the API key is still attached. When null a + /// default handler is created and owned by this instance. + /// + internal BraintrustOpenApiClient( + BraintrustConfig config, + HttpMessageHandler? innerHandler, + Func? btqlDelayFunc = null, + bool noBtqlDelay = false) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + + var authHandler = new BearerTokenHandler(config, innerHandler ?? new HttpClientHandler()); + _httpClient = new HttpClient(authHandler, disposeHandler: innerHandler is null) + { + BaseAddress = new Uri(config.ApiUrl), + Timeout = config.RequestTimeout, + }; + + _api = new Generated.BraintrustGeneratedApiClient(_httpClient) + { + BaseUrl = config.ApiUrl, + + // The generated client deserializes straight off the response stream by + // default and keeps no copy of the body. Reading it as a string first preserves + // the server's diagnostic on the generated exception. + ReadResponseAsString = true, + }; + _btqlClient = new BtqlClient(config, _httpClient, btqlDelayFunc, noBtqlDelay); + } + + /// + /// The generated OpenAPI client, wired up with this instance's base URL, API key and + /// timeout. Use it for the Braintrust REST endpoints this SDK does not wrap - see + /// docs/api-client.md. + /// + /// + /// This is raw generated code, so it is shaped by the spec rather than by the SDK: + /// property names are spec-cased (Org_id), identifiers are , + /// list endpoints take their filters as positional arguments, and failures surface as + /// rather than . It also + /// tracks whatever spec ref the build pinned, so it is not covered by the SDK's own + /// compatibility promises. + /// + /// The returned client shares this instance's , so it stops + /// working once this instance is disposed. + /// + public Generated.IBraintrustGeneratedApiClient Api => _api; + + internal Task>> QuerySpansAsync( + string experimentId, + string rootTraceId, + IReadOnlyCollection expectedSpanIds, + CancellationToken cancellationToken = default) + => _btqlClient.QuerySpansAsync(experimentId, rootTraceId, expectedSpanIds, cancellationToken); + + internal Task>> QueryAsync( + string query, + CancellationToken cancellationToken = default) + => _btqlClient.QueryAsync(query, cancellationToken); + + /// + /// Resolve the configured project, optionally creating a named project when it is absent. + /// An explicit id wins over the configured id and name. + /// + internal async Task FetchProjectAsync( + string? projectId = null, + bool createIfMissing = true, + CancellationToken cancellationToken = default) + { + projectId ??= _config.DefaultProjectId; + if (projectId is not null) + { + if (!Guid.TryParse(projectId, out var id)) + { + throw new InvalidOperationException($"Invalid project id: {projectId}"); + } + + try + { + return await _api.GetProjectIdAsync(id, cancellationToken).ConfigureAwait(false); + } + catch (Generated.ApiException ex) when (ex.StatusCode == (int)HttpStatusCode.NotFound) + { + throw new InvalidOperationException($"Invalid project id: {projectId}"); + } + } + + var projectName = _config.DefaultProjectName; + if (string.IsNullOrEmpty(projectName)) + { + throw new InvalidOperationException( + "Either DefaultProjectId or DefaultProjectName must be set in config"); + } + + return await FetchProjectByNameAsync(projectName, createIfMissing, cancellationToken) + .ConfigureAwait(false); + } + + internal async Task FetchProjectByNameAsync( + string projectName, + bool createIfMissing, + CancellationToken cancellationToken = default) + { + var page = await _api.GetProjectAsync( + limit: 2, + starting_after: null, + ending_before: null, + ids: null, + project_name: projectName, + org_name: null, + cancellationToken).ConfigureAwait(false); + + return page.Objects.Count switch + { + 0 when createIfMissing => await _api.PostProjectAsync( + new Generated.CreateProject { Name = projectName }, + cancellationToken).ConfigureAwait(false), + 0 => throw new InvalidOperationException($"Project '{projectName}' not found"), + > 1 => throw new InvalidOperationException( + $"Found {page.Objects.Count} projects named '{projectName}'; " + + "use a project id to disambiguate"), + _ => page.Objects.First(), + }; + } + + internal async Task<(Generated.Project Project, Generated.Organization Organization)> + FetchProjectAndOrgAsync( + string? projectId = null, + CancellationToken cancellationToken = default) + { + var project = await FetchProjectAsync(projectId, cancellationToken: cancellationToken) + .ConfigureAwait(false); + var organization = await _api.GetOrganizationIdAsync(project.Org_id, cancellationToken) + .ConfigureAwait(false); + return (project, organization); + } + + internal async Task FetchProjectUriAsync(CancellationToken cancellationToken = default) + { + var (project, organization) = await FetchProjectAndOrgAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + return BuildAppUri("app", organization.Name, "p", project.Name); + } + + internal Uri BuildExperimentUri( + Generated.Organization organization, + Generated.Project project, + string experimentName) + => BuildAppUri("app", organization.Name, "p", project.Name, "experiments", experimentName); + + private Uri BuildAppUri(params string[] segments) + { + var baseUri = new Uri(_config.AppUrl); + var prefix = baseUri.GetLeftPart(UriPartial.Path).TrimEnd('/'); + var path = string.Join("/", segments.Select(Uri.EscapeDataString)); + return new Uri($"{prefix}/{path}", UriKind.Absolute); + } + + public void Dispose() => _httpClient.Dispose(); +} diff --git a/src/Braintrust.Sdk/Api/DefaultBraintrustApiClient.cs b/src/Braintrust.Sdk/Api/DefaultBraintrustApiClient.cs index e9784aa..afbbbfc 100644 --- a/src/Braintrust.Sdk/Api/DefaultBraintrustApiClient.cs +++ b/src/Braintrust.Sdk/Api/DefaultBraintrustApiClient.cs @@ -1,32 +1,19 @@ using System.Net; -using System.Net.Http.Json; using System.Text.Json; -using System.Text.Json.Serialization; using Braintrust.Sdk.Config; using Generated = Braintrust.Sdk.Api.Generated; namespace Braintrust.Sdk.Api; /// -/// Default implementation, backed by the client -/// generated from the Braintrust OpenAPI spec. -/// -/// The generated types are an implementation detail: they use spec-shaped names -/// (Org_id) and identifiers, so this class maps them onto the -/// SDK's own records. Login is the one endpoint absent from the spec, so it stays -/// hand-written here - the same split sdk-java makes. +/// Use instead. /// +[Obsolete("Use BraintrustOpenApiClient instead.")] public sealed class DefaultBraintrustApiClient : IBraintrustApiClient, IDisposable { - private static readonly JsonSerializerOptions LoginJsonOptions = new() - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - PropertyNameCaseInsensitive = true, - }; - private readonly BraintrustConfig _config; - private readonly HttpClient _httpClient; - private readonly Generated.BraintrustGeneratedApiClient _api; + private readonly BraintrustOpenApiClient _client; + private readonly bool _ownsClient; public static DefaultBraintrustApiClient Of(BraintrustConfig config) => new(config); @@ -35,63 +22,30 @@ public DefaultBraintrustApiClient(BraintrustConfig config) { } - /// - /// Transport to send through, wrapped so the API key is still attached. When null a - /// default handler is created and owned by this instance. - /// internal DefaultBraintrustApiClient(BraintrustConfig config, HttpMessageHandler? innerHandler) + : this(config, new BraintrustOpenApiClient(config, innerHandler), ownsClient: true) { - _config = config ?? throw new ArgumentNullException(nameof(config)); - - var authHandler = new BearerTokenHandler(config, innerHandler ?? new HttpClientHandler()); - _httpClient = new HttpClient(authHandler, disposeHandler: innerHandler is null) - { - BaseAddress = new Uri(config.ApiUrl), - Timeout = config.RequestTimeout, - }; - - _api = new Generated.BraintrustGeneratedApiClient(_httpClient) - { - BaseUrl = config.ApiUrl, + } - // The generated client deserializes straight off the response stream by - // default and keeps no copy of the body, so a failure reaches Translate below - // with an empty Response - reporting "status 400:" and nothing else. Reading - // the body as a string first keeps the server's diagnostic, which is also how - // the previous hand-rolled client behaved. - ReadResponseAsString = true, - }; + internal DefaultBraintrustApiClient( + BraintrustConfig config, + BraintrustOpenApiClient client, + bool ownsClient) + { + _config = config ?? throw new ArgumentNullException(nameof(config)); + _client = client ?? throw new ArgumentNullException(nameof(client)); + _ownsClient = ownsClient; } /// - /// The generated OpenAPI client, wired up with this instance's base URL, API key and - /// timeout. Use it for the Braintrust REST endpoints this SDK does not wrap - see - /// docs/api-client.md. + /// The generated OpenAPI client owned by this facade. New code should use + /// instead. /// - /// - /// This is raw generated code, so it is shaped by the spec rather than by the SDK: - /// property names are spec-cased (Org_id), identifiers are , - /// list endpoints take their filters as positional arguments, and failures surface as - /// rather than . It also - /// tracks whatever spec ref the build pinned, so it is not covered by the SDK's own - /// compatibility promises. Prefer the methods on - /// where they cover what you need. - /// - /// The returned client shares this instance's , so it stops - /// working once this instance is disposed. - /// - public Generated.IBraintrustGeneratedApiClient Api => _api; + public Generated.IBraintrustGeneratedApiClient Api => _client.Api; public async Task GetOrCreateProject(string projectName) - { - // POST /v1/project upserts by name. - var created = await Send(() => _api.PostProjectAsync(new Generated.CreateProject - { - Name = projectName, - })).ConfigureAwait(false); - - return ToProject(created); - } + => ToProject(await _client.FetchProjectByNameAsync(projectName, createIfMissing: true) + .ConfigureAwait(false)); public async Task GetProject(string projectId) { @@ -99,16 +53,12 @@ public async Task GetOrCreateProject(string projectName) try { - return ToProject(await _api.GetProjectIdAsync(id).ConfigureAwait(false)); + return ToProject(await Api.GetProjectIdAsync(id).ConfigureAwait(false)); } catch (Generated.ApiException ex) when (ex.StatusCode == (int)HttpStatusCode.NotFound) { return null; } - catch (Generated.ApiException ex) - { - throw Translate(ex); - } } public async Task GetOrCreateExperiment(CreateExperimentRequest request) @@ -130,18 +80,17 @@ public async Task GetOrCreateExperiment(CreateExperimentRequest requ body.Base_exp_id = ParseId(request.BaseExperimentId, nameof(request.BaseExperimentId)); } - var created = await Send(() => _api.PostExperimentAsync(body)).ConfigureAwait(false); - return ToExperiment(created); + return ToExperiment(await Api.PostExperimentAsync(body).ConfigureAwait(false)); } public async Task GetProjectAndOrgInfo() { - if (_config.DefaultProjectId != null) + if (_config.DefaultProjectId is not null) { return await GetProjectAndOrgInfo(_config.DefaultProjectId).ConfigureAwait(false); } - if (_config.DefaultProjectName != null) + if (_config.DefaultProjectName is not null) { var project = await GetOrCreateProject(_config.DefaultProjectName).ConfigureAwait(false); return await GetProjectAndOrgInfo(project.Id).ConfigureAwait(false); @@ -153,24 +102,22 @@ public async Task GetOrCreateExperiment(CreateExperimentRequest requ public async Task GetProjectAndOrgInfo(string projectId) { var project = await GetProject(projectId).ConfigureAwait(false); - if (project == null) - { - return null; - } - - return new OrganizationAndProjectInfo(await ResolveOrg(project).ConfigureAwait(false), project); + return project is null + ? null + : new OrganizationAndProjectInfo( + await ResolveOrg(project).ConfigureAwait(false), project); } public async Task GetOrCreateProjectAndOrgInfo() { Project project; - if (_config.DefaultProjectId != null) + if (_config.DefaultProjectId is not null) { project = await GetProject(_config.DefaultProjectId).ConfigureAwait(false) ?? throw new ApiException($"Project with ID {_config.DefaultProjectId} not found"); } - else if (_config.DefaultProjectName != null) + else if (_config.DefaultProjectName is not null) { project = await GetOrCreateProject(_config.DefaultProjectName).ConfigureAwait(false); } @@ -185,64 +132,11 @@ public async Task GetOrCreateProjectAndOrgInfo() private async Task ResolveOrg(Project project) { - var login = await Login().ConfigureAwait(false); - var orgInfo = login.OrgInfo.FirstOrDefault(org => - string.Equals(org.Id, project.OrgId, StringComparison.OrdinalIgnoreCase)); - - return orgInfo - ?? throw new ApiException($"Organization {project.OrgId} not found for project {project.Id}"); - } - - /// - /// /api/apikey/login is not part of the OpenAPI spec, so it is issued directly. - /// - private async Task Login() - { - var apiKey = await _config.GetRequiredApiKeyAsync().ConfigureAwait(false); - - using var response = await _httpClient - .PostAsJsonAsync("/api/apikey/login", new LoginRequest(apiKey), LoginJsonOptions) + var organization = await Api.GetOrganizationIdAsync(Guid.Parse(project.OrgId)) .ConfigureAwait(false); - - var content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - { - throw new ApiException( - (int)response.StatusCode, - $"API request failed with status {(int)response.StatusCode}: {content}"); - } - - return JsonSerializer.Deserialize(content, LoginJsonOptions) - ?? throw new ApiException("Failed to deserialize API response"); - } - - private static async Task Send(Func> call) - { - try - { - return await call().ConfigureAwait(false); - } - catch (Generated.ApiException ex) - { - throw Translate(ex); - } + return new OrganizationInfo(organization.Id.ToString(), organization.Name); } - /// - /// Maps a generated-client failure onto the SDK's own exception. Response holds - /// the server's body; a few paths (an empty body, an unexpected status) leave it blank, - /// so fall back to the generated message rather than reporting a bare status code. - /// - private static ApiException Translate(Generated.ApiException ex) - { - var detail = string.IsNullOrWhiteSpace(ex.Response) ? ex.Message : ex.Response; - return new ApiException(ex.StatusCode, $"API request failed with status {ex.StatusCode}: {detail}"); - } - - /// - /// The spec types identifiers as UUIDs, so the generated client does too. Surface a - /// malformed one as an API error rather than a FormatException from deep inside. - /// private static Guid ParseId(string value, string name) => Guid.TryParse(value, out var id) ? id @@ -265,12 +159,6 @@ private static Guid ParseId(string value, string name) => Created: experiment.Created?.ToString("o"), Updated: ExtensionString(experiment.AdditionalProperties, "updated")); - /// - /// Reads a field the spec does not declare out of the generated type's extension - /// data. updated is one of these: the SDK's records expose it and the previous - /// client picked it up straight off the wire, so it is recovered here rather than - /// silently becoming null. - /// private static string? ExtensionString(IDictionary? extensionData, string name) { if (extensionData is null || !extensionData.TryGetValue(name, out var value) || value is null) @@ -314,5 +202,11 @@ private static Guid ParseId(string value, string name) => CommitTime: repoInfo.Commit_time, GitDiff: repoInfo.Git_diff); - public void Dispose() => _httpClient.Dispose(); + public void Dispose() + { + if (_ownsClient) + { + _client.Dispose(); + } + } } diff --git a/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs b/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs index bb603f2..9a39aa3 100644 --- a/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs +++ b/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs @@ -1,8 +1,9 @@ namespace Braintrust.Sdk.Api; /// -/// Interface for Braintrust API client operations. +/// Use and the generated OpenAPI models instead. /// +[Obsolete("Use BraintrustOpenApiClient.Api and the Braintrust.Sdk.Api.Generated models instead.")] public interface IBraintrustApiClient { /// diff --git a/src/Braintrust.Sdk/Api/Internal/BtqlClient.cs b/src/Braintrust.Sdk/Api/Internal/BtqlClient.cs index 9920c6d..2ff067a 100644 --- a/src/Braintrust.Sdk/Api/Internal/BtqlClient.cs +++ b/src/Braintrust.Sdk/Api/Internal/BtqlClient.cs @@ -8,23 +8,21 @@ namespace Braintrust.Sdk.Api.Internal; /// /// Internal client for querying the Braintrust BTQL API. -/// Retries with exponential backoff until spans are fresh (freshness == "complete") or max retries are hit. /// -internal class BtqlClient : IBtqlClient +internal sealed class BtqlClient { - private const int MaxFreshnessRetries = 7; - private const int BaseFreshnessDelayMs = 1000; - private const int MaxFreshnessDelayMs = 8000; + private const int MaxAttempts = 8; + private const int BaseDelayMs = 1000; + private const int MaxDelayMs = 8000; private readonly BraintrustConfig _config; private readonly HttpClient _httpClient; - private readonly bool _ownsHttpClient; private readonly JsonSerializerOptions _jsonOptions; private readonly Func _delayFunc; internal BtqlClient( BraintrustConfig config, - HttpClient? httpClient = null, + HttpClient httpClient, Func? delayFunc = null, bool noDelay = false) { @@ -39,59 +37,67 @@ internal BtqlClient( DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; - if (httpClient == null) - { - _httpClient = new HttpClient { BaseAddress = new Uri(config.ApiUrl) }; - _ownsHttpClient = true; - } - else - { - _httpClient = httpClient; - _ownsHttpClient = false; - } + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); } /// - /// Queries spans for a given experiment and root span ID via the BTQL API. - /// Retries until freshness is "complete" and rows are non-empty, or until max retries are hit. - /// Backoff schedule: 1s, 2s, 4s, 8s, 8s, 8s, 8s (7 delays, 8 total attempts). + /// Queries spans for a given experiment and root trace ID via the BTQL API. + /// Retries until every expected span is present, or throws after eight attempts. + /// Backoff schedule: 1s, 2s, 4s, 8s, 8s, 8s, 8s. /// Score-type spans are excluded from results. /// - public async Task>> QuerySpansAsync( - string experimentId, string rootSpanId, CancellationToken cancellationToken = default) + internal async Task>> QuerySpansAsync( + string experimentId, + string rootTraceId, + IReadOnlyCollection expectedSpanIds, + CancellationToken cancellationToken = default) { var safeExperimentId = experimentId.Replace("'", "''"); - var safeRootSpanId = rootSpanId.Replace("'", "''"); - var query = $"SELECT * FROM experiment('{safeExperimentId}') WHERE root_span_id = '{safeRootSpanId}' AND span_attributes.type != 'score' LIMIT 1000"; + var safeRootTraceId = rootTraceId.Replace("'", "''"); + var query = $"SELECT * FROM experiment('{safeExperimentId}') WHERE root_span_id = '{safeRootTraceId}' AND span_attributes.type != 'score' LIMIT 1000"; - BtqlResponse? lastResponse = null; - int delayMs = BaseFreshnessDelayMs; + int delayMs = BaseDelayMs; - for (int attempt = 0; attempt <= MaxFreshnessRetries; attempt++) + for (int attempt = 0; attempt < MaxAttempts; attempt++) { if (attempt > 0) { await _delayFunc(delayMs, cancellationToken).ConfigureAwait(false); - delayMs = Math.Min(delayMs * 2, MaxFreshnessDelayMs); + delayMs = Math.Min(delayMs * 2, MaxDelayMs); } - lastResponse = await PostBtqlAsync(query, cancellationToken).ConfigureAwait(false); + var response = await PostBtqlAsync(query, cancellationToken).ConfigureAwait(false); - if ((lastResponse.Freshness == "complete" && lastResponse.Data.Count > 0) - || attempt == MaxFreshnessRetries) + var presentSpanIds = response.Data + .Select(row => row.TryGetValue("span_id", out var id) ? id.GetString() : null) + .Where(id => id is not null) + .ToHashSet(StringComparer.Ordinal); + var missingSpanIds = expectedSpanIds + .Where(id => !presentSpanIds.Contains(id)) + .ToList(); + + if (missingSpanIds.Count == 0) { - break; + return response.Data + .Cast>() + .ToList(); } - } - if (lastResponse == null || lastResponse.Data.Count == 0) - { - return []; + if (attempt == MaxAttempts - 1) + { + throw new InvalidOperationException( + $"Timed out waiting for trace spans: {string.Join(", ", missingSpanIds)}"); + } } - return lastResponse.Data - .Cast>() - .ToList(); + throw new InvalidOperationException("BTQL trace query exhausted its retry loop"); + } + + internal async Task>> QueryAsync( + string query, CancellationToken cancellationToken = default) + { + var response = await PostBtqlAsync(query, cancellationToken).ConfigureAwait(false); + return response.Data.Cast>().ToList(); } private async Task PostBtqlAsync(string query, CancellationToken cancellationToken) @@ -109,14 +115,6 @@ private async Task PostBtqlAsync(string query, CancellationToken c return JsonSerializer.Deserialize(content, _jsonOptions) ?? new BtqlResponse(); } - public void Dispose() - { - if (_ownsHttpClient) - { - _httpClient.Dispose(); - } - } - private record BtqlRequest( [property: JsonPropertyName("query")] string Query); @@ -125,7 +123,25 @@ private class BtqlResponse [JsonPropertyName("data")] public List> Data { get; init; } = new(); - [JsonPropertyName("freshness")] - public string? Freshness { get; init; } + [JsonPropertyName("freshness_state")] + public FreshnessState? FreshnessState { get; init; } + + [JsonPropertyName("realtime_state")] + public RealtimeState? RealtimeState { get; init; } + } + + private sealed class FreshnessState + { + [JsonPropertyName("last_processed_xact_id")] + public string? LastProcessedXactId { get; init; } + + [JsonPropertyName("last_considered_xact_id")] + public string? LastConsideredXactId { get; init; } + } + + private sealed class RealtimeState + { + [JsonPropertyName("type")] + public string? Type { get; init; } } } diff --git a/src/Braintrust.Sdk/Api/Internal/IBtqlClient.cs b/src/Braintrust.Sdk/Api/Internal/IBtqlClient.cs deleted file mode 100644 index 3056b3f..0000000 --- a/src/Braintrust.Sdk/Api/Internal/IBtqlClient.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Text.Json; - -namespace Braintrust.Sdk.Api.Internal; - -internal interface IBtqlClient : IDisposable -{ - Task>> QuerySpansAsync( - string experimentId, string rootSpanId, CancellationToken cancellationToken = default); -} diff --git a/src/Braintrust.Sdk/Api/Models.cs b/src/Braintrust.Sdk/Api/Models.cs index a08a8d4..b70750c 100644 --- a/src/Braintrust.Sdk/Api/Models.cs +++ b/src/Braintrust.Sdk/Api/Models.cs @@ -1,11 +1,13 @@ using System.Text.Json.Serialization; using Braintrust.Sdk.Git; +using Generated = Braintrust.Sdk.Api.Generated; namespace Braintrust.Sdk.Api; /// -/// Represents a Braintrust project. +/// Use through instead. /// +[Obsolete("Use BraintrustOpenApiClient.Api and Braintrust.Sdk.Api.Generated.Project instead.")] public record Project( [property: JsonPropertyName("id")] string Id, [property: JsonPropertyName("name")] string Name, @@ -15,8 +17,9 @@ public record Project( ); /// -/// Represents a Braintrust experiment. +/// Use through instead. /// +[Obsolete("Use BraintrustOpenApiClient.Api and Braintrust.Sdk.Api.Generated.Experiment instead.")] public record Experiment( [property: JsonPropertyName("id")] string Id, [property: JsonPropertyName("project_id")] string ProjectId, @@ -29,31 +32,41 @@ public record Experiment( ); /// -/// Represents organization information. +/// Use through +/// instead. /// +[Obsolete("Use BraintrustOpenApiClient.Api and Braintrust.Sdk.Api.Generated.Organization instead.")] public record OrganizationInfo( [property: JsonPropertyName("id")] string Id, [property: JsonPropertyName("name")] string Name ); /// -/// Combined organization and project information. +/// Use and through +/// instead. /// +[Obsolete("Use BraintrustOpenApiClient.Api with Braintrust.Sdk.Api.Generated.Organization and Project instead.")] public record OrganizationAndProjectInfo( OrganizationInfo OrgInfo, Project Project ); /// -/// Request to create a project. +/// Use with +/// +/// instead. /// +[Obsolete("Use BraintrustOpenApiClient.Api.PostProjectAsync with Braintrust.Sdk.Api.Generated.CreateProject instead.")] public record CreateProjectRequest( [property: JsonPropertyName("name")] string Name ); /// -/// Request to create an experiment. +/// Use with +/// +/// instead. /// +[Obsolete("Use BraintrustOpenApiClient.Api.PostExperimentAsync with Braintrust.Sdk.Api.Generated.CreateExperiment instead.")] public record CreateExperimentRequest( [property: JsonPropertyName("project_id")] string ProjectId, [property: JsonPropertyName("name")] string Name, @@ -63,17 +76,3 @@ public record CreateExperimentRequest( [property: JsonPropertyName("tags")] IReadOnlyList? Tags = null, [property: JsonPropertyName("metadata")] IReadOnlyDictionary? Metadata = null ); - -/// -/// Login request with API key. -/// -internal record LoginRequest( - [property: JsonPropertyName("token")] string Token -); - -/// -/// Response from login endpoint. -/// -internal record LoginResponse( - [property: JsonPropertyName("org_info")] List OrgInfo -); diff --git a/src/Braintrust.Sdk/Braintrust.cs b/src/Braintrust.Sdk/Braintrust.cs index f61aef2..edb91f7 100644 --- a/src/Braintrust.Sdk/Braintrust.cs +++ b/src/Braintrust.Sdk/Braintrust.cs @@ -84,18 +84,37 @@ internal static void ResetForTest() /// public static Braintrust Of(BraintrustConfig config, bool autoManageOpenTelemetry = true) { - var apiClient = DefaultBraintrustApiClient.Of(config); - return new Braintrust(config, apiClient, autoManageOpenTelemetry); + return new Braintrust(config, BraintrustOpenApiClient.Of(config), autoManageOpenTelemetry); } + /// + /// Create a new Braintrust instance backed by the given api client. Primarily useful for testing. + /// + internal static Braintrust Of( + BraintrustConfig config, BraintrustOpenApiClient apiClient, bool autoManageOpenTelemetry = false) + => new(config, apiClient, autoManageOpenTelemetry); + public BraintrustConfig Config { get; } - public IBraintrustApiClient ApiClient { get; } + + /// + /// The client used for all Braintrust API requests. + /// + public BraintrustOpenApiClient OpenApiClient => _apiClient; + + /// + /// Use instead. + /// + [Obsolete("Use OpenApiClient instead.")] + public IBraintrustApiClient ApiClient => + new DefaultBraintrustApiClient(Config, _apiClient, ownsClient: false); + + private readonly BraintrustOpenApiClient _apiClient; private volatile OpenTelemetry.Trace.TracerProvider? _tracer; - private Braintrust(BraintrustConfig config, IBraintrustApiClient apiClient, bool autoManageOpenTelemetry) + private Braintrust(BraintrustConfig config, BraintrustOpenApiClient apiClient, bool autoManageOpenTelemetry) { Config = config ?? throw new ArgumentNullException(nameof(config)); - ApiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); + _apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); if (autoManageOpenTelemetry) { _tracer = Trace.BraintrustTracing.CreateTracerProvider(this.Config); @@ -105,11 +124,12 @@ private Braintrust(BraintrustConfig config, IBraintrustApiClient apiClient, bool /// /// Get the URI to the configured Braintrust org and project. /// - public async Task GetProjectUriAsync() - { - var orgAndProject = await ApiClient.GetOrCreateProjectAndOrgInfo().ConfigureAwait(false); - return new Uri($"{Config.AppUrl}/app/{orgAndProject.OrgInfo.Name}/p/{orgAndProject.Project.Name}"); - } + /// + /// Use when turning the result into a string: + /// unescapes the path, which breaks the link for org or project + /// names containing a space. + /// + public Task GetProjectUriAsync() => _apiClient.FetchProjectUriAsync(); /// /// Add Braintrust to existing OpenTelemetry TracerProviderBuilder. @@ -135,6 +155,54 @@ public System.Diagnostics.ActivitySource GetActivitySource() return Trace.BraintrustTracing.GetActivitySource(); } + /// + /// Fetch a dataset by name from the configured default project. + /// + /// Rows are fetched lazily, so this only resolves the project and the dataset id. Each case's + /// input and expected are deserialized into and + /// as they are read. + /// + /// Handing the result to links the experiment to + /// this dataset and each eval row back to the record it came from. + /// + /// Name of the dataset within the configured project. + /// + /// Transaction id to pin to. Null resolves the latest version at the start of each + /// enumeration. + /// + /// + /// Reads a row's input. Null deserializes it into . + /// + /// + /// Reads a row's expected, which the dataset schema leaves optional. Null + /// deserializes it into , so a dataset with rows that have no + /// expected needs a converter here to be read at all. + /// + public async Task> FetchDatasetAsync( + string datasetName, + string? version = null, + Func? inputConverter = null, + Func? expectedConverter = null) + where TInput : notnull + where TOutput : notnull + { + // Handles both BRAINTRUST_DEFAULT_PROJECT_ID and _NAME. This is a read, so a name that + // matches no project fails rather than creating one. + var project = await _apiClient + .FetchProjectAsync(createIfMissing: false) + .ConfigureAwait(false); + + // By id, not name: a name is only unique within an org, and an api key can span orgs. + return await Eval.Dataset.FetchByProjectIdAsync( + // Reuses this instance's caller-owned client, and so its connection pool. + _apiClient, + project.Id.ToString(), + datasetName, + version, + inputConverter, + expectedConverter).ConfigureAwait(false); + } + /// /// Create a new eval builder. /// @@ -144,6 +212,6 @@ public Eval.Eval.Builder EvalBuilder() { return Eval.Eval.NewBuilder() .Config(Config) - .ApiClient(ApiClient); + .ApiClient(_apiClient); } } diff --git a/src/Braintrust.Sdk/Eval/DatasetBrainstoreImpl.cs b/src/Braintrust.Sdk/Eval/DatasetBrainstoreImpl.cs new file mode 100644 index 0000000..1e849f2 --- /dev/null +++ b/src/Braintrust.Sdk/Eval/DatasetBrainstoreImpl.cs @@ -0,0 +1,231 @@ +using System.Runtime.CompilerServices; +using System.Text.Json; +using Braintrust.Sdk.Api; +using Generated = Braintrust.Sdk.Api.Generated; + +namespace Braintrust.Sdk.Eval; + +/// +/// A dataset read from Braintrust, one page at a time, through the generated OpenAPI client. +/// +/// Enumerating this is what sdk-java calls opening a cursor: the version is resolved once, up +/// front, and every page is then read as of that version, so a dataset written to mid-run still +/// yields one consistent snapshot. There is no Cursor type because C# already has one - +/// GetAsyncEnumerator opens, MoveNextAsync advances, DisposeAsync closes. +/// +internal sealed class DatasetBrainstoreImpl : IDataset + where TInput : notnull + where TOutput : notnull +{ + /// Rows per fetch. Matches sdk-java. + private const int BatchSize = 512; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + }; + + /// Stands in for a field the row left out. Detached, so nothing owns a document. + private static readonly JsonElement NullElement = JsonSerializer.SerializeToElement(null); + + private readonly BraintrustOpenApiClient _apiClient; + private readonly string? _pinnedVersion; + private readonly Func _inputConverter; + private readonly Func _expectedConverter; + + internal DatasetBrainstoreImpl( + BraintrustOpenApiClient apiClient, + string datasetId, + string? version = null, + Func? inputConverter = null, + Func? expectedConverter = null) + { + _apiClient = apiClient; + Id = datasetId; + _pinnedVersion = version; + _inputConverter = inputConverter ?? (element => Deserialize(element, "input")); + _expectedConverter = expectedConverter ?? (element => Deserialize(element, "expected")); + } + + public string Id { get; } + + public string? Version => _pinnedVersion; + + /// + /// Returns the iterator rather than being one itself, so that a token supplied through + /// WithCancellation binds to 's parameter and reaches the + /// requests below. An iterator method with no such parameter would ignore it. + /// + public IAsyncEnumerable> GetCasesAsync() => IterateAsync(); + + private async IAsyncEnumerable> IterateAsync( + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var version = _pinnedVersion ?? await ResolveLatestVersionAsync(cancellationToken).ConfigureAwait(false); + if (version == null) + { + // Empty dataset - nothing to read, and no version to read it at. + yield break; + } + + await foreach (var datasetCase in GetCasesAtVersionAsync(version, cancellationToken).ConfigureAwait(false)) + { + yield return datasetCase; + } + } + + internal async Task OpenSnapshotAsync( + CancellationToken cancellationToken = default) + { + var version = _pinnedVersion ?? await ResolveLatestVersionAsync(cancellationToken).ConfigureAwait(false); + return new Snapshot( + version, + version is null ? EmptyAsync() : GetCasesAtVersionAsync(version, cancellationToken)); + } + + internal sealed record Snapshot( + string? Version, + IAsyncEnumerable> Cases); + + private static async IAsyncEnumerable> EmptyAsync() + { + await Task.CompletedTask; + yield break; + } + + private async IAsyncEnumerable> GetCasesAtVersionAsync( + string version, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + string? cursor = null; + var seenIds = new HashSet(); + do + { + var page = await _apiClient.Api.PostDatasetIdFetchAsync( + Guid.Parse(Id), + new Generated.FetchEventsRequest + { + Limit = BatchSize, + Cursor = cursor, + Version = version, + }, + cancellationToken).ConfigureAwait(false); + + foreach (var row in page.Events) + { + // Fetch pages run newest to oldest over event history, so an updated row can + // appear again on a later page with an older transaction id. + if (seenIds.Add(row.Id)) + { + yield return ToCase(row); + } + } + + // The API omits the cursor once the result set is empty. + cursor = page.Events.Count == 0 || string.IsNullOrEmpty(page.Cursor) ? null : page.Cursor; + } + while (cursor != null); + } + + /// + /// The max transaction id in the dataset, or null if the dataset is empty. This is the + /// snapshot every page of one enumeration is then read against. BTQL rather than the + /// generated client because POST /btql is not in the OpenAPI spec. + /// + private async Task ResolveLatestVersionAsync(CancellationToken cancellationToken) + { + var safeId = Id.Replace("'", "''"); + var rows = await _apiClient + .QueryAsync($"SELECT max(_xact_id) AS version, count(*) AS count FROM dataset('{safeId}')", cancellationToken) + .ConfigureAwait(false); + + if (rows.Count == 0) + { + throw new InvalidOperationException($"Failed to resolve version for dataset {Id}: empty response"); + } + + var row = rows[0]; + if (row.TryGetValue("count", out var count) && count.ToString() == "0") + { + return null; + } + + return row.TryGetValue("version", out var version) && version.ValueKind != JsonValueKind.Null + ? version.ToString() + : throw new InvalidOperationException($"Failed to resolve version for dataset {Id}"); + } + + private DatasetCase ToCase(Generated.DatasetEvent row) + { + return new DatasetCase( + Convert(row, "input", row.Input, _inputConverter), + Convert(row, "expected", row.Expected, _expectedConverter), + row.Tags?.ToList() ?? [], + ToMetadata(row.Metadata)) + { + Origin = new Origin( + ObjectType: "dataset", + ObjectId: row.Dataset_id.ToString(), + Id: row.Id, + XactId: row._xact_id, + Created: row.Created.ToString("o")), + }; + } + + /// + /// The spec gives dataset metadata one declared property (model) and leaves the rest + /// free-form, so the values arrive split across the class and its extension data. Flatten + /// both back into the single bag a carries. + /// + private static IReadOnlyDictionary ToMetadata(Generated.Metadata? metadata) + { + if (metadata is null) + { + return new Dictionary(); + } + + var merged = new Dictionary(metadata.AdditionalProperties); + if (metadata.Model is not null) + { + merged["model"] = metadata.Model; + } + + return merged; + } + + /// + /// Hand one field of a row to its converter. expected is optional in the dataset + /// schema, and a row that omits a field - or stores it as JSON null - arrives here as a null + /// object, so both are normalized to a JSON null element: TInput and TOutput are + /// non-nullable and the default converters cannot represent that, but a caller-supplied + /// converter still gets to decide what a missing field means. + /// + private T Convert(Generated.DatasetEvent row, string field, object? value, Func converter) + { + var element = value switch + { + JsonElement raw => raw, + null => NullElement, + // The generated client types these as object, so anything it did not leave as a + // JsonElement is round-tripped rather than handed to the converter as-is. + _ => JsonSerializer.SerializeToElement(value, JsonOptions), + }; + + try + { + return converter(element); + } + catch (Exception ex) when (ex is InvalidOperationException or JsonException or NotSupportedException) + { + throw new InvalidOperationException( + $"Dataset row {row.Id} in dataset {Id}: could not read '{field}' as {typeof(T).Name}. " + + "Datasets with rows the default deserializer cannot handle - a missing 'expected', " + + "say - can be read with the inputConverter/expectedConverter overloads.", + ex); + } + } + + private static T Deserialize(JsonElement element, string field) + => element.Deserialize(JsonOptions) + ?? throw new InvalidOperationException($"Dataset row '{field}' deserialized to null"); +} diff --git a/src/Braintrust.Sdk/Eval/DatasetCase.cs b/src/Braintrust.Sdk/Eval/DatasetCase.cs index 890e1ea..e8d88a1 100644 --- a/src/Braintrust.Sdk/Eval/DatasetCase.cs +++ b/src/Braintrust.Sdk/Eval/DatasetCase.cs @@ -50,6 +50,12 @@ public record DatasetCase public required IReadOnlyList Tags { get; init; } public required IReadOnlyDictionary Metadata { get; init; } + /// + /// Where this case was read from, when it came from a Braintrust dataset. Null for + /// in-memory cases. + /// + public Origin? Origin { get; init; } + [SetsRequiredMembers] public DatasetCase( TInput input, diff --git a/src/Braintrust.Sdk/Eval/DatasetInMemoryImpl.cs b/src/Braintrust.Sdk/Eval/DatasetInMemoryImpl.cs index be3260d..fdeac63 100644 --- a/src/Braintrust.Sdk/Eval/DatasetInMemoryImpl.cs +++ b/src/Braintrust.Sdk/Eval/DatasetInMemoryImpl.cs @@ -17,7 +17,7 @@ public DatasetInMemoryImpl(IEnumerable> cases) public string Id { get; } - public string Version => "0"; + public string? Version => "0"; #pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously public async IAsyncEnumerable> GetCasesAsync() diff --git a/src/Braintrust.Sdk/Eval/Eval.cs b/src/Braintrust.Sdk/Eval/Eval.cs index d57843c..1af838e 100644 --- a/src/Braintrust.Sdk/Eval/Eval.cs +++ b/src/Braintrust.Sdk/Eval/Eval.cs @@ -1,10 +1,12 @@ using System.Diagnostics; +using System.Runtime.ExceptionServices; using System.Text.Json; +using System.Threading.Channels; using Braintrust.Sdk.Api; -using Braintrust.Sdk.Api.Internal; using Braintrust.Sdk.Config; using Braintrust.Sdk.Git; using Braintrust.Sdk.Trace; +using Generated = Braintrust.Sdk.Api.Generated; namespace Braintrust.Sdk.Eval; @@ -13,7 +15,7 @@ namespace Braintrust.Sdk.Eval; /// /// The type of input data for the evaluation /// The type of output produced by the task -public sealed class Eval +public sealed class Eval : IDisposable where TInput : notnull where TOutput : notnull { @@ -24,10 +26,9 @@ public sealed class Eval }; private readonly string _experimentName; - private readonly BraintrustConfig _config; - private readonly IBraintrustApiClient _client; - private readonly IBtqlClient _btqlClient; - private readonly OrganizationAndProjectInfo _orgAndProject; + private readonly BraintrustOpenApiClient _apiClient; + private readonly Generated.Project _project; + private readonly Generated.Organization _organization; private readonly ActivitySource _activitySource; private readonly IDataset _dataset; private readonly ITask _task; @@ -38,13 +39,21 @@ public sealed class Eval private readonly int? _maxConcurrency; private readonly RepoInfo? _repoInfo; - private Eval(Builder builder, OrganizationAndProjectInfo orgAndProject, RepoInfo? repoInfo) + /// The client this eval created, or null when the caller owns it. + private readonly BraintrustOpenApiClient? _ownedApiClient; + + private Eval( + Builder builder, + BraintrustOpenApiClient apiClient, + Generated.Project project, + Generated.Organization organization, + RepoInfo? repoInfo, + BraintrustOpenApiClient? ownedApiClient) { _experimentName = builder._experimentName; - _config = builder._config ?? throw new ArgumentNullException(nameof(builder._config)); - _client = builder._apiClient ?? throw new ArgumentNullException(nameof(builder._apiClient)); - _btqlClient = builder._btqlClient ?? throw new ArgumentNullException(nameof(builder._btqlClient)); - _orgAndProject = orgAndProject ?? throw new ArgumentNullException(nameof(orgAndProject)); + _apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); + _project = project ?? throw new ArgumentNullException(nameof(project)); + _organization = organization ?? throw new ArgumentNullException(nameof(organization)); _activitySource = builder._activitySource ?? throw new ArgumentNullException(nameof(builder._activitySource)); _dataset = builder._dataset ?? throw new ArgumentNullException(nameof(builder._dataset)); @@ -55,6 +64,16 @@ private Eval(Builder builder, OrganizationAndProjectInfo orgAndProject, RepoInfo _experimentMetadata = builder._experimentMetadata; _maxConcurrency = builder._maxConcurrency; _repoInfo = repoInfo; + _ownedApiClient = ownedApiClient; + } + + /// + /// Disposes the http clients this eval created for itself. Clients passed to the builder are + /// left alone. + /// + public void Dispose() + { + _ownedApiClient?.Dispose(); } /// @@ -62,50 +81,213 @@ private Eval(Builder builder, OrganizationAndProjectInfo orgAndProject, RepoInfo /// public async Task RunAsync() { - var experiment = await _client.GetOrCreateExperiment( - new CreateExperimentRequest( - _orgAndProject.Project.Id, - _experimentName, - RepoInfo: _repoInfo, - Tags: _experimentTags, - Metadata: _experimentMetadata)) - .ConfigureAwait(false); + IAsyncEnumerable> cases; + string? datasetVersion = null; + + var isRemote = _dataset is DatasetBrainstoreImpl; + if (_dataset is DatasetBrainstoreImpl remoteDataset) + { + var snapshot = await remoteDataset.OpenSnapshotAsync().ConfigureAwait(false); + cases = snapshot.Cases; + datasetVersion = snapshot.Version; + } + else + { + cases = _dataset.GetCasesAsync(); + } - var experimentId = experiment.Id; + // Only Braintrust datasets have an id the API will accept; the in-memory id is a + // local placeholder. - var cases = new List>(); - await foreach (var datasetCase in _dataset.GetCasesAsync()) + // POST /v1/experiment upserts by name within the project. + var body = new Generated.CreateExperiment { - cases.Add(datasetCase); + Project_id = _project.Id, + Name = _experimentName, + Repo_info = ToGeneratedRepoInfo(_repoInfo), + Tags = _experimentTags?.ToList(), + Metadata = _experimentMetadata?.ToDictionary(kv => kv.Key, kv => kv.Value), + }; + + if (isRemote) + { + body.Dataset_id = Guid.Parse(_dataset.Id); + body.Dataset_version = datasetVersion; } + var experiment = await _apiClient.Api.PostExperimentAsync(body).ConfigureAwait(false); + + var experimentId = experiment.Id.ToString(); + // Run cases in parallel if (_maxConcurrency.HasValue) { - using var semaphore = new SemaphoreSlim(_maxConcurrency.Value); - var tasks = cases.Select(async datasetCase => + await RunCasesWithBoundedConcurrency( + experimentId, + cases, + _maxConcurrency.Value).ConfigureAwait(false); + } + else + { + await RunCasesWithUnlimitedConcurrency(experimentId, cases).ConfigureAwait(false); + } + + var experimentUrl = _apiClient + .BuildExperimentUri(_organization, _project, experiment.Name ?? _experimentName) + .AbsoluteUri; + return new EvalResult(experimentUrl); + } + + private async Task RunCasesWithBoundedConcurrency( + string experimentId, + IAsyncEnumerable> cases, + int maxConcurrency) + { + var queue = Channel.CreateBounded>( + new BoundedChannelOptions(maxConcurrency) + { + SingleWriter = true, + FullMode = BoundedChannelFullMode.Wait, + }); + using var stop = new CancellationTokenSource(); + + async Task ProduceAsync() + { + try + { + await using var enumerator = cases + .WithCancellation(stop.Token) + .ConfigureAwait(false) + .GetAsyncEnumerator(); + + // Wait for queue capacity before advancing the dataset cursor. This prevents + // fetching another page while both the workers and queue are full. + while (await queue.Writer.WaitToWriteAsync(stop.Token).ConfigureAwait(false) + && await enumerator.MoveNextAsync()) + { + if (!queue.Writer.TryWrite(enumerator.Current)) + { + await queue.Writer.WriteAsync(enumerator.Current, stop.Token).ConfigureAwait(false); + } + } + + queue.Writer.TryComplete(); + } + catch (OperationCanceledException) when (stop.IsCancellationRequested) + { + queue.Writer.TryComplete(); + } + catch (Exception ex) + { + stop.Cancel(); + queue.Writer.TryComplete(ex); + throw; + } + } + + async Task ConsumeAsync() + { + while (await queue.Reader.WaitToReadAsync().ConfigureAwait(false)) { - await semaphore.WaitAsync().ConfigureAwait(false); + if (stop.IsCancellationRequested) + { + return; + } + + if (!queue.Reader.TryRead(out var datasetCase)) + { + continue; + } + try { await EvalOne(experimentId, datasetCase).ConfigureAwait(false); } - finally + catch { - semaphore.Release(); + // Stop dataset enumeration immediately. Other workers still finish and are + // observed before the failure is propagated by WhenAll below. + stop.Cancel(); + throw; } - }); - await Task.WhenAll(tasks).ConfigureAwait(false); + } } - else + + var producer = ProduceAsync(); + var workers = Enumerable.Range(0, maxConcurrency) + .Select(_ => ConsumeAsync()) + .ToArray(); + + try + { + await Task.WhenAll(workers.Prepend(producer)).ConfigureAwait(false); + } + finally + { + stop.Cancel(); + } + } + + private async Task RunCasesWithUnlimitedConcurrency( + string experimentId, + IAsyncEnumerable> cases) + { + var tasks = new List(); + Exception? enumerationException = null; + Exception? caseException = null; + using var stop = new CancellationTokenSource(); + + async Task RunCaseAsync(DatasetCase datasetCase) + { + try + { + await EvalOne(experimentId, datasetCase).ConfigureAwait(false); + } + catch (Exception ex) + { + Interlocked.CompareExchange(ref caseException, ex, null); + stop.Cancel(); + throw; + } + } + + try + { + await foreach (var datasetCase in cases + .WithCancellation(stop.Token) + .ConfigureAwait(false)) + { + if (stop.IsCancellationRequested) + { + break; + } + tasks.Add(RunCaseAsync(datasetCase)); + } + } + catch (OperationCanceledException) when ( + stop.IsCancellationRequested && Volatile.Read(ref caseException) != null) + { + // A case failed and canceled dataset production. + } + catch (Exception ex) + { + enumerationException = ex; + } + + try { - // Unlimited parallelism - var tasks = cases.Select(datasetCase => EvalOne(experimentId, datasetCase)); await Task.WhenAll(tasks).ConfigureAwait(false); } + catch when (enumerationException != null) + { + // All case tasks have been observed. Prefer the dataset failure that stopped + // enumeration over a concurrent case failure. + } - var experimentUrl = CreateExperimentUrl(_config.AppUrl, _orgAndProject, _experimentName); - return new EvalResult(experimentUrl); + if (enumerationException != null) + { + ExceptionDispatchInfo.Capture(enumerationException).Throw(); + } } private async Task EvalOne(string experimentId, DatasetCase datasetCase) @@ -127,6 +309,11 @@ private async Task EvalOne(string experimentId, DatasetCase dat rootActivity.SetTag("braintrust.input_json", ToJson(datasetCase.Input)); rootActivity.SetTag("braintrust.expected_json", ToJson(datasetCase.Expected)); + if (datasetCase.Origin != null) + { + rootActivity.SetTag("braintrust.origin", ToJson(datasetCase.Origin)); + } + if (datasetCase.Tags.Count > 0) { // Use native string array attribute (not JSON) to match Go/Java SDKs @@ -143,8 +330,10 @@ private async Task EvalOne(string experimentId, DatasetCase dat // Run task TaskResult? taskResult = null; Exception? taskException = null; + string? taskSpanId; { - var taskActivity = _activitySource.StartActivity("task"); + using var taskActivity = _activitySource.StartActivity("task"); + taskSpanId = taskActivity?.SpanId.ToHexString(); taskActivity?.SetTag(BraintrustTracing.ParentKey, $"experiment_id:{experimentId}"); taskActivity?.SetTag("braintrust.span_attributes", ToJson(new { type = "task" })); taskActivity?.SetTag("braintrust.input_json", ToJson(datasetCase.Input)); @@ -182,8 +371,12 @@ private async Task EvalOne(string experimentId, DatasetCase dat } // Create a lazy trace object backed by BTQL (only queries API when first accessed) - var rootSpanId = rootActivity.TraceId.ToHexString(); - var trace = new EvalTrace(ct => _btqlClient.QuerySpansAsync(experimentId, rootSpanId, ct)); + var rootTraceId = rootActivity.TraceId.ToHexString(); + var trace = new EvalTrace(ct => _apiClient.QuerySpansAsync( + experimentId, + rootTraceId, + taskSpanId is null ? [] : [taskSpanId], + ct)); await RunScorersAndClassifiers(experimentId, rootActivity, taskResult!.Value, trace, datasetCase.Metadata) .ConfigureAwait(false); @@ -572,23 +765,19 @@ private static ActivityEvent CreateExceptionEvent(Exception ex) return new ActivityEvent("exception", tags: tags); } - private static string CreateExperimentUrl( - string appUrl, - OrganizationAndProjectInfo orgAndProject, - string experimentName) - { - var baseUri = new Uri(appUrl); - var path = string.Join("/", - "app", - Uri.EscapeDataString(orgAndProject.OrgInfo.Name), - "p", - Uri.EscapeDataString(orgAndProject.Project.Name), - "experiments", - Uri.EscapeDataString(experimentName)); - - return new UriBuilder(baseUri.Scheme, baseUri.Host, baseUri.Port, "/" + path).Uri.AbsoluteUri; - } - + private static Generated.RepoInfo? ToGeneratedRepoInfo(RepoInfo? repoInfo) => + repoInfo is null ? null : new Generated.RepoInfo + { + Commit = repoInfo.Commit, + Branch = repoInfo.Branch, + Tag = repoInfo.Tag, + Dirty = repoInfo.Dirty, + Author_name = repoInfo.AuthorName, + Author_email = repoInfo.AuthorEmail, + Commit_message = repoInfo.CommitMessage, + Commit_time = repoInfo.CommitTime, + Git_diff = repoInfo.GitDiff, + }; /// /// Creates a new eval builder. @@ -605,8 +794,8 @@ public sealed class Builder { internal string _experimentName = "unnamed-dotnet-eval"; internal BraintrustConfig? _config; - internal IBraintrustApiClient? _apiClient; - internal IBtqlClient? _btqlClient; + internal BraintrustOpenApiClient? _apiClient; + internal string? _projectId; internal ActivitySource? _activitySource; internal IDataset? _dataset; @@ -628,9 +817,9 @@ public async Task> BuildAsync() _config ??= BraintrustConfig.FromEnvironment(); _activitySource ??= BraintrustTracing.GetActivitySource(); _projectId ??= _config.DefaultProjectId; - _apiClient ??= DefaultBraintrustApiClient.Of(_config); - _btqlClient ??= new BtqlClient(_config); + // Validate before opening any connection, so a misconfigured builder does not leak + // an HttpClient on the way out. if (_scorers.Count == 0 && _classifiers.Count == 0) { throw new InvalidOperationException("Must provide at least one scorer or classifier"); @@ -646,32 +835,44 @@ public async Task> BuildAsync() throw new InvalidOperationException("Must provide a task"); } - OrganizationAndProjectInfo? orgAndProject; - - if (_projectId == null) + // BraintrustOpenApiClient owns both generated OpenAPI calls and the internal BTQL + // calls. Anything created here is handed to the Eval to dispose - and the builder's + // own fields stay untouched, so building twice remains safe. + var apiClient = _apiClient; + BraintrustOpenApiClient? ownedApiClient = null; + if (apiClient is null) { - orgAndProject = await _apiClient.GetProjectAndOrgInfo().ConfigureAwait(false) - ?? throw new InvalidOperationException("Unable to retrieve project and org info"); - } - else - { - orgAndProject = await _apiClient.GetProjectAndOrgInfo(_projectId).ConfigureAwait(false) - ?? throw new InvalidOperationException($"Invalid project id: {_projectId}"); + apiClient = BraintrustOpenApiClient.Of(_config); + ownedApiClient = apiClient; } - // Collect git repo info: use explicit value if set, otherwise auto-detect. - // This is intentionally non-throwing — if git is unavailable, repoInfo is simply null. - RepoInfo? repoInfo; - if (_repoInfoExplicitlySet) + try { - repoInfo = _repoInfo; + var (project, organization) = await apiClient + .FetchProjectAndOrgAsync(_projectId) + .ConfigureAwait(false); + + // Collect git repo info: use explicit value if set, otherwise auto-detect. + // This is intentionally non-throwing — if git is unavailable, repoInfo is simply null. + RepoInfo? repoInfo; + if (_repoInfoExplicitlySet) + { + repoInfo = _repoInfo; + } + else + { + repoInfo = await GitUtil.GetRepoInfoAsync(_gitMetadataSettings).ConfigureAwait(false); + } + + return new Eval( + this, apiClient, project, organization, repoInfo, ownedApiClient); } - else + catch { - repoInfo = await GitUtil.GetRepoInfoAsync(_gitMetadataSettings).ConfigureAwait(false); + // Nothing takes ownership if the build fails, so close what was opened here. + ownedApiClient?.Dispose(); + throw; } - - return new Eval(this, orgAndProject, repoInfo); } /// @@ -702,21 +903,11 @@ public Builder Config(BraintrustConfig config) } /// - /// Set the API client. - /// - public Builder ApiClient(IBraintrustApiClient apiClient) - { - _apiClient = apiClient; - return this; - } - - /// - /// Set the BTQL client (used to retrieve trace spans for ITracedScorer). - /// Primarily useful for testing. + /// Set the API client to run against. /// - internal Builder BtqlClient(IBtqlClient btqlClient) + public Builder ApiClient(BraintrustOpenApiClient apiClient) { - _btqlClient = btqlClient; + _apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); return this; } diff --git a/src/Braintrust.Sdk/Eval/IDataset.cs b/src/Braintrust.Sdk/Eval/IDataset.cs index b0a8686..dc3ac25 100644 --- a/src/Braintrust.Sdk/Eval/IDataset.cs +++ b/src/Braintrust.Sdk/Eval/IDataset.cs @@ -1,3 +1,6 @@ +using System.Text.Json; +using Braintrust.Sdk.Api; + namespace Braintrust.Sdk.Eval; /// @@ -13,7 +16,12 @@ public interface IDataset where TOutput : notnull { /// - /// Open a cursor to iterate through dataset cases. + /// Open a cursor to iterate through dataset cases. Remote implementations fetch lazily, + /// so enumerating this is what issues requests. + /// + /// To cancel a long read, enumerate with + /// - the token reaches the + /// requests the remote implementation makes. /// IAsyncEnumerable> GetCasesAsync(); @@ -23,9 +31,10 @@ public interface IDataset string Id { get; } /// - /// Gets the dataset version. + /// The pinned dataset version (transaction id). Null means the dataset is unpinned and + /// resolves the latest version independently each time it is enumerated. /// - string Version { get; } + string? Version { get; } } /// @@ -44,4 +53,168 @@ public static IDataset Of(params DatasetCase(cases); } -} \ No newline at end of file + + /// + /// Fetch a dataset from Braintrust by project and dataset name. + /// + /// The name is resolved to an id here, so a bad name fails now rather than part-way through + /// an eval. Rows themselves are fetched lazily, a page at a time, each time the returned + /// dataset is enumerated. + /// + /// Caller-owned client to read through. + /// Project containing the dataset. + /// Name of the dataset within that project. + /// + /// Transaction id to pin to. Null resolves the latest version at the start of each + /// enumeration, so every row of a given run is read from one consistent snapshot. + /// + /// + /// Reads a row's input. Null deserializes it into . + /// + /// + /// Reads a row's expected. Null deserializes it into . + /// + /// No dataset by that name exists in that project. + public static async Task> FetchFromBraintrustAsync( + BraintrustOpenApiClient apiClient, + string projectName, + string datasetName, + string? version = null, + Func? inputConverter = null, + Func? expectedConverter = null) + where TInput : notnull + where TOutput : notnull + { + ArgumentNullException.ThrowIfNull(apiClient); + + var datasetId = await ResolveDatasetIdAsync( + apiClient, datasetName, CancellationToken.None, projectName: projectName) + .ConfigureAwait(false); + + return NewRemote(apiClient, datasetId, version, inputConverter, expectedConverter); + } + + /// + /// Same as , but scoped by project id. + /// This is what the SDK itself uses: a project name is only unique within an org, so an api + /// key that spans orgs can resolve one name to two different projects. + /// + internal static async Task> FetchByProjectIdAsync( + BraintrustOpenApiClient apiClient, + string projectId, + string datasetName, + string? version = null, + Func? inputConverter = null, + Func? expectedConverter = null, + CancellationToken cancellationToken = default) + where TInput : notnull + where TOutput : notnull + { + ArgumentNullException.ThrowIfNull(apiClient); + + var datasetId = await ResolveDatasetIdAsync( + apiClient, datasetName, cancellationToken, projectId: projectId) + .ConfigureAwait(false); + + return NewRemote(apiClient, datasetId, version, inputConverter, expectedConverter); + } + + /// + /// Fetch a dataset from Braintrust by id. Unlike + /// this makes no request until the + /// dataset is enumerated. + /// + /// Caller-owned client to read through. + /// Braintrust dataset id. + /// Transaction id to pin to, or null for the latest. + /// + /// Reads a row's input. Null deserializes it into . + /// + /// + /// Reads a row's expected. Null deserializes it into . + /// + /// is not a valid id. + public static IDataset FromId( + BraintrustOpenApiClient apiClient, + string datasetId, + string? version = null, + Func? inputConverter = null, + Func? expectedConverter = null) + where TInput : notnull + where TOutput : notnull + { + ArgumentNullException.ThrowIfNull(apiClient); + return NewRemote(apiClient, datasetId, version, inputConverter, expectedConverter); + } + + /// + /// Build a remote dataset over the caller's API client. The client also owns the internal + /// BTQL support used to resolve an unpinned dataset's version. + /// + private static IDataset NewRemote( + BraintrustOpenApiClient apiClient, + string datasetId, + string? version, + Func? inputConverter, + Func? expectedConverter) + where TInput : notnull + where TOutput : notnull + { + // Every read of the dataset addresses it by id, so a malformed one should fail here + // rather than as a FormatException part-way through an enumeration. + if (!Guid.TryParse(datasetId, out _)) + { + throw new ArgumentException($"Invalid dataset id: {datasetId}", nameof(datasetId)); + } + + return new DatasetBrainstoreImpl( + apiClient, + datasetId, + version, + inputConverter, + expectedConverter); + } + + private static async Task ResolveDatasetIdAsync( + BraintrustOpenApiClient apiClient, + string datasetName, + CancellationToken cancellationToken, + string? projectId = null, + string? projectName = null) + { + Guid? projectGuid = null; + if (projectId is not null) + { + if (!Guid.TryParse(projectId, out var parsed)) + { + throw new InvalidOperationException($"Invalid project id: {projectId}"); + } + projectGuid = parsed; + } + else if (string.IsNullOrEmpty(projectName)) + { + throw new InvalidOperationException("Either a project id or a project name is required"); + } + + var page = await apiClient.Api.GetDatasetAsync( + limit: 2, + starting_after: null, + ending_before: null, + ids: null, + dataset_name: datasetName, + project_name: projectGuid is null ? projectName : null, + project_id: projectGuid, + org_name: null, + cancellationToken).ConfigureAwait(false); + + var project = projectGuid?.ToString() ?? projectName; + return page.Objects.Count switch + { + 0 => throw new InvalidOperationException( + $"Dataset '{datasetName}' not found in project '{project}'"), + > 1 => throw new InvalidOperationException( + $"Expected one dataset named '{datasetName}' in project '{project}', found {page.Objects.Count}"), + _ => page.Objects.First().Id.ToString(), + }; + } +} diff --git a/src/Braintrust.Sdk/Origin.cs b/src/Braintrust.Sdk/Origin.cs new file mode 100644 index 0000000..484e806 --- /dev/null +++ b/src/Braintrust.Sdk/Origin.cs @@ -0,0 +1,21 @@ +using System.Text.Json.Serialization; + +namespace Braintrust.Sdk; + +/// +/// A pointer to the Braintrust object a row came from - for eval rows read out of a dataset, +/// the dataset and the specific record within it. Logged as braintrust.origin so the UI +/// can link an experiment row back to its source. +/// +/// Kind of object pointed at, e.g. dataset. +/// Id of that object, e.g. the dataset id. +/// Id of the item within it, e.g. the dataset row id. +/// Transaction id the item was read at. +/// Creation timestamp of the item. +public record Origin( + [property: JsonPropertyName("object_type")] string ObjectType, + [property: JsonPropertyName("object_id")] string ObjectId, + [property: JsonPropertyName("id")] string Id, + [property: JsonPropertyName("_xact_id")] string XactId, + [property: JsonPropertyName("created")] string Created +); diff --git a/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs b/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs deleted file mode 100644 index 7ef606c..0000000 --- a/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs +++ /dev/null @@ -1,213 +0,0 @@ -using System.Net; -using System.Text; -using System.Text.Json; -using Braintrust.Sdk.Api; -using Braintrust.Sdk.Config; - -// BraintrustApiClient is deprecated in favour of DefaultBraintrustApiClient, but it is -// still shipped, so it stays under test until it is removed. -#pragma warning disable CS0618 - -namespace Braintrust.Sdk.Tests.Api; - -[Collection("BraintrustGlobals")] -public class BraintrustApiClientTest : IDisposable -{ - private readonly TestHttpMessageHandler _handler; - private readonly HttpClient _httpClient; - private readonly BraintrustApiClient _apiClient; - - public BraintrustApiClientTest() - { - _handler = new TestHttpMessageHandler(); - _httpClient = new HttpClient(_handler) - { - BaseAddress = new Uri("https://test-api.example.com") - }; - - var config = BraintrustConfig.Of( - ("BRAINTRUST_API_KEY", "test-api-key"), - ("BRAINTRUST_API_URL", "https://test-api.example.com"), - ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") - ); - - _apiClient = new BraintrustApiClient(config, _httpClient); - } - - public void Dispose() - { - _apiClient.Dispose(); - _httpClient.Dispose(); - _handler.Dispose(); - } - - [Fact] - public async Task GetOrCreateProject_CreatesProject() - { - var expectedProject = new Project("proj-123", "test-project", "org-456"); - _handler.SetResponse(HttpStatusCode.OK, expectedProject); - - var result = await _apiClient.GetOrCreateProject("test-project"); - - Assert.NotNull(result); - Assert.Equal("proj-123", result.Id); - Assert.Equal("test-project", result.Name); - Assert.Equal("org-456", result.OrgId); - - // Verify request - Assert.Equal(HttpMethod.Post, _handler.LastRequest?.Method); - Assert.Equal("/v1/project", _handler.LastRequest?.RequestUri?.AbsolutePath); - Assert.Equal("Bearer test-api-key", _handler.LastRequest?.Headers.Authorization?.ToString()); - } - - [Fact] - public async Task GetProject_ReturnsProject() - { - var expectedProject = new Project("proj-123", "test-project", "org-456"); - _handler.SetResponse(HttpStatusCode.OK, expectedProject); - - var result = await _apiClient.GetProject("proj-123"); - - Assert.NotNull(result); - Assert.Equal("proj-123", result.Id); - Assert.Equal(HttpMethod.Get, _handler.LastRequest?.Method); - Assert.Equal("/v1/project/proj-123", _handler.LastRequest?.RequestUri?.AbsolutePath); - } - - [Fact] - public async Task GetProject_ReturnsNull_When404() - { - _handler.SetResponse(HttpStatusCode.NotFound, "Not found"); - - var result = await _apiClient.GetProject("missing-project"); - - Assert.Null(result); - } - - [Fact] - public async Task GetOrCreateExperiment_CreatesExperiment() - { - var expectedExperiment = new Experiment("exp-123", "proj-456", "test-experiment"); - _handler.SetResponse(HttpStatusCode.OK, expectedExperiment); - - var request = new CreateExperimentRequest("proj-456", "test-experiment", "Test description"); - var result = await _apiClient.GetOrCreateExperiment(request); - - Assert.NotNull(result); - Assert.Equal("exp-123", result.Id); - Assert.Equal("proj-456", result.ProjectId); - Assert.Equal("test-experiment", result.Name); - Assert.Equal(HttpMethod.Post, _handler.LastRequest?.Method); - Assert.Equal("/v1/experiment", _handler.LastRequest?.RequestUri?.AbsolutePath); - } - - [Fact] - public async Task GetOrCreateProjectAndOrgInfo_ReturnsInfo() - { - // Setup: First call creates/gets project, second call is login - var project = new Project("proj-123", "test-project", "org-456"); - var loginResponse = new LoginResponse([new OrganizationInfo("org-456", "Test Org")]); - - _handler.SetResponses( - (HttpStatusCode.OK, project), - (HttpStatusCode.OK, loginResponse) - ); - - var result = await _apiClient.GetOrCreateProjectAndOrgInfo(); - - Assert.NotNull(result); - Assert.Equal("proj-123", result.Project.Id); - Assert.Equal("org-456", result.OrgInfo.Id); - Assert.Equal("Test Org", result.OrgInfo.Name); - } - - [Fact] - public async Task ApiException_ThrownOn_HttpError() - { - _handler.SetResponse(HttpStatusCode.BadRequest, "Bad request"); - - var exception = await Assert.ThrowsAsync(() => - _apiClient.GetProject("test")); - - Assert.Equal(400, exception.StatusCode); - Assert.Contains("400", exception.Message); - } - - [Fact] - public async Task UsesApiKeyFromBraintrustEnvFile() - { - var originalCwd = Directory.GetCurrentDirectory(); - var originalApiKey = Environment.GetEnvironmentVariable("BRAINTRUST_API_KEY"); - var tempDir = Directory.CreateTempSubdirectory("braintrust-api-client-env-").FullName; - - try - { - Environment.SetEnvironmentVariable("BRAINTRUST_API_KEY", null); - File.WriteAllText(Path.Combine(tempDir, ".env.braintrust"), "BRAINTRUST_API_KEY=file-api-key\n"); - Directory.SetCurrentDirectory(tempDir); - - using var handler = new TestHttpMessageHandler(); - using var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://test-api.example.com") - }; - - var config = BraintrustConfig.Of( - ("BRAINTRUST_API_URL", "https://test-api.example.com"), - ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") - ); - using var apiClient = new BraintrustApiClient(config, httpClient); - handler.SetResponse(HttpStatusCode.OK, new Project("proj-123", "test-project", "org-456")); - - await apiClient.GetOrCreateProject("test-project"); - - Assert.Equal("Bearer file-api-key", handler.LastRequest?.Headers.Authorization?.ToString()); - } - finally - { - Directory.SetCurrentDirectory(originalCwd); - Environment.SetEnvironmentVariable("BRAINTRUST_API_KEY", originalApiKey); - Directory.Delete(tempDir, recursive: true); - } - } - - // Test HttpMessageHandler for mocking HTTP responses - private class TestHttpMessageHandler : HttpMessageHandler - { - private Queue<(HttpStatusCode, object)> _responses = new(); - public HttpRequestMessage? LastRequest { get; private set; } - - public void SetResponse(HttpStatusCode statusCode, object response) - { - _responses.Enqueue((statusCode, response)); - } - - public void SetResponses(params (HttpStatusCode, object)[] responses) - { - _responses = new Queue<(HttpStatusCode, object)>(responses); - } - - protected override async Task SendAsync( - HttpRequestMessage request, - CancellationToken cancellationToken) - { - LastRequest = request; - - if (_responses.Count == 0) - { - throw new InvalidOperationException("No response configured for test"); - } - - var (statusCode, responseObj) = _responses.Dequeue(); - var json = responseObj is string str ? str : JsonSerializer.Serialize(responseObj, new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower - }); - - return await Task.FromResult(new HttpResponseMessage(statusCode) - { - Content = new StringContent(json, Encoding.UTF8, "application/json") - }); - } - } -} diff --git a/tests/Braintrust.Sdk.Tests/Api/BraintrustOpenApiClientTest.cs b/tests/Braintrust.Sdk.Tests/Api/BraintrustOpenApiClientTest.cs new file mode 100644 index 0000000..edb54b1 --- /dev/null +++ b/tests/Braintrust.Sdk.Tests/Api/BraintrustOpenApiClientTest.cs @@ -0,0 +1,142 @@ +using System.Net; +using System.Text; +using Braintrust.Sdk.Api; +using Braintrust.Sdk.Config; +using Generated = Braintrust.Sdk.Api.Generated; + +namespace Braintrust.Sdk.Tests.Api; + +[Collection("BraintrustGlobals")] +public class BraintrustOpenApiClientTest : IDisposable +{ + private const string ProjectId = "3f2504e0-4f89-11d3-9a0c-0305e82c3301"; + private const string OrgId = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; + + private readonly StubHandler _handler; + private readonly BraintrustOpenApiClient _apiClient; + + public BraintrustOpenApiClientTest() + { + _handler = new StubHandler(); + _apiClient = new BraintrustOpenApiClient(Config(), _handler); + } + + public void Dispose() + { + _apiClient.Dispose(); + _handler.Dispose(); + } + + private static BraintrustConfig Config() => BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-api-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project")); + + private static string ProjectJson => + $$"""{"id":"{{ProjectId}}","org_id":"{{OrgId}}","name":"test-project"}"""; + + [Fact] + public async Task Uses_api_key_from_the_braintrust_env_file() + { + var originalCwd = Directory.GetCurrentDirectory(); + var originalApiKey = Environment.GetEnvironmentVariable("BRAINTRUST_API_KEY"); + var tempDir = Directory.CreateTempSubdirectory("braintrust-open-api-client-env-").FullName; + + try + { + Environment.SetEnvironmentVariable("BRAINTRUST_API_KEY", null); + File.WriteAllText(Path.Combine(tempDir, ".env.braintrust"), + "BRAINTRUST_API_KEY=file-api-key\n"); + Directory.SetCurrentDirectory(tempDir); + + using var handler = new StubHandler(); + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_URL", "https://test-api.example.com")); + using var apiClient = new BraintrustOpenApiClient(config, handler); + handler.Enqueue(HttpStatusCode.OK, $$"""{"objects":[{{ProjectJson}}]}"""); + + await apiClient.Api.GetProjectAsync( + limit: 1, + starting_after: null, + ending_before: null, + ids: null, + project_name: "test-project", + org_name: null); + + Assert.Equal("Bearer file-api-key", handler.LastRequest?.Headers.Authorization?.ToString()); + } + finally + { + Directory.SetCurrentDirectory(originalCwd); + Environment.SetEnvironmentVariable("BRAINTRUST_API_KEY", originalApiKey); + Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public void Braintrust_resolves_to_the_openapi_client_by_default() + { + var braintrust = Braintrust.Of(Config(), autoManageOpenTelemetry: false); + + Assert.IsType(braintrust.OpenApiClient); + } + + [Fact] + public async Task Api_exposes_the_generated_client_wired_to_this_config() + { + _handler.Enqueue(HttpStatusCode.OK, $$"""{"objects":[{{ProjectJson}}]}"""); + + var page = await _apiClient.Api.GetProjectAsync( + limit: 1, + starting_after: null, + ending_before: null, + ids: null, + project_name: "test-project", + org_name: null); + + var project = Assert.Single(page.Objects); + Assert.Equal(Guid.Parse(ProjectId), project.Id); + Assert.Equal("test-api.example.com", _handler.LastRequest?.RequestUri?.Host); + Assert.Equal("/v1/project", _handler.LastRequest?.RequestUri?.AbsolutePath); + Assert.Contains("project_name=test-project", _handler.LastRequest?.RequestUri?.Query); + Assert.Equal("Bearer test-api-key", _handler.LastRequest?.Headers.Authorization?.ToString()); + } + + [Fact] + public async Task Api_surfaces_failures_as_the_generated_exception() + { + _handler.Enqueue(HttpStatusCode.NotFound, "no such project"); + + var exception = await Assert.ThrowsAnyAsync( + () => _apiClient.Api.GetProjectIdAsync(Guid.Parse(ProjectId))); + + Assert.Equal(404, exception.StatusCode); + Assert.Contains("no such project", exception.Response); + } + + private sealed class StubHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _responses = new(); + + public HttpRequestMessage? LastRequest { get; private set; } + + public void Enqueue(HttpStatusCode status, string body) => _responses.Enqueue((status, body)); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + LastRequest = request; + + if (_responses.Count == 0) + { + throw new InvalidOperationException("No response configured for test"); + } + + var (status, body) = _responses.Dequeue(); + return Task.FromResult(new HttpResponseMessage(status) + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }); + } + } +} diff --git a/tests/Braintrust.Sdk.Tests/Api/DefaultBraintrustApiClientTest.cs b/tests/Braintrust.Sdk.Tests/Api/DefaultBraintrustApiClientTest.cs deleted file mode 100644 index f6f5e74..0000000 --- a/tests/Braintrust.Sdk.Tests/Api/DefaultBraintrustApiClientTest.cs +++ /dev/null @@ -1,327 +0,0 @@ -using System.Net; -using System.Text; -using Braintrust.Sdk.Api; -using Braintrust.Sdk.Config; -using Braintrust.Sdk.Git; -using Generated = Braintrust.Sdk.Api.Generated; - -namespace Braintrust.Sdk.Tests.Api; - -/// -/// Covers , which fulfils -/// through the generated OpenAPI client. Responses are -/// written as raw JSON rather than serialized from the SDK's records, so these assert -/// against the actual wire shape the API returns. -/// -[Collection("BraintrustGlobals")] -public class DefaultBraintrustApiClientTest : IDisposable -{ - private const string ProjectId = "3f2504e0-4f89-11d3-9a0c-0305e82c3301"; - private const string OrgId = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; - private const string ExperimentId = "17c1a5a0-1234-4c56-8def-0123456789ab"; - - private readonly StubHandler _handler; - private readonly DefaultBraintrustApiClient _apiClient; - - public DefaultBraintrustApiClientTest() - { - _handler = new StubHandler(); - - var config = BraintrustConfig.Of( - ("BRAINTRUST_API_KEY", "test-api-key"), - ("BRAINTRUST_API_URL", "https://test-api.example.com"), - ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") - ); - - _apiClient = new DefaultBraintrustApiClient(config, _handler); - } - - public void Dispose() - { - _apiClient.Dispose(); - _handler.Dispose(); - } - - private static string ProjectJson => - $$"""{"id":"{{ProjectId}}","org_id":"{{OrgId}}","name":"test-project"}"""; - - [Fact] - public async Task GetOrCreateProject_posts_and_maps_the_project() - { - _handler.Enqueue(HttpStatusCode.OK, ProjectJson); - - var project = await _apiClient.GetOrCreateProject("test-project"); - - Assert.Equal(ProjectId, project.Id); - Assert.Equal("test-project", project.Name); - Assert.Equal(OrgId, project.OrgId); - - Assert.Equal(HttpMethod.Post, _handler.LastRequest?.Method); - Assert.Equal("/v1/project", _handler.LastRequest?.RequestUri?.AbsolutePath); - Assert.Equal("Bearer test-api-key", _handler.LastRequest?.Headers.Authorization?.ToString()); - Assert.Contains("\"name\":\"test-project\"", _handler.LastRequestBody); - } - - [Fact] - public async Task GetProject_gets_by_id() - { - _handler.Enqueue(HttpStatusCode.OK, ProjectJson); - - var project = await _apiClient.GetProject(ProjectId); - - Assert.NotNull(project); - Assert.Equal(ProjectId, project!.Id); - Assert.Equal(HttpMethod.Get, _handler.LastRequest?.Method); - Assert.Equal($"/v1/project/{ProjectId}", _handler.LastRequest?.RequestUri?.AbsolutePath); - } - - [Fact] - public async Task GetProject_returns_null_on_404() - { - _handler.Enqueue(HttpStatusCode.NotFound, "Not found"); - - Assert.Null(await _apiClient.GetProject(ProjectId)); - } - - [Fact] - public async Task GetOrCreateExperiment_posts_the_request_and_maps_the_result() - { - _handler.Enqueue(HttpStatusCode.OK, - $$"""{"id":"{{ExperimentId}}","project_id":"{{ProjectId}}","name":"test-experiment"}"""); - - var experiment = await _apiClient.GetOrCreateExperiment( - new CreateExperimentRequest(ProjectId, "test-experiment", "Test description")); - - Assert.Equal(ExperimentId, experiment.Id); - Assert.Equal(ProjectId, experiment.ProjectId); - Assert.Equal("test-experiment", experiment.Name); - - Assert.Equal(HttpMethod.Post, _handler.LastRequest?.Method); - Assert.Equal("/v1/experiment", _handler.LastRequest?.RequestUri?.AbsolutePath); - Assert.Contains($"\"project_id\":\"{ProjectId}\"", _handler.LastRequestBody); - Assert.Contains("\"name\":\"test-experiment\"", _handler.LastRequestBody); - } - - [Fact] - public async Task GetOrCreateExperiment_sends_repo_info_and_reads_it_back() - { - _handler.Enqueue(HttpStatusCode.OK, - $$""" - {"repo_info":{"commit":"abc123","branch":"main","dirty":false,"author_name":"Ada"}, - "id":"{{ExperimentId}}","project_id":"{{ProjectId}}","name":"e"} - """); - - var experiment = await _apiClient.GetOrCreateExperiment(new CreateExperimentRequest( - ProjectId, "e", - RepoInfo: new RepoInfo(Commit: "abc123", Branch: "main", Dirty: false, AuthorName: "Ada"), - Tags: ["nightly"], - Metadata: new Dictionary { ["run"] = 7 })); - - // Request side: the SDK's RepoInfo maps onto the generated body. - Assert.Contains("\"commit\":\"abc123\"", _handler.LastRequestBody); - Assert.Contains("\"author_name\":\"Ada\"", _handler.LastRequestBody); - Assert.Contains("\"tags\":[\"nightly\"]", _handler.LastRequestBody); - - // Response side: it maps back onto the SDK's RepoInfo. - Assert.Equal("abc123", experiment.RepoInfo?.Commit); - Assert.Equal("main", experiment.RepoInfo?.Branch); - Assert.Equal("Ada", experiment.RepoInfo?.AuthorName); - } - - [Fact] - public async Task Maps_fields_the_spec_does_not_declare() - { - // `updated` is absent from the spec but exposed by the SDK's records, and the - // previous client read it straight off the wire. It survives via extension data. - _handler.Enqueue(HttpStatusCode.OK, - $$""" - {"id":"{{ProjectId}}","org_id":"{{OrgId}}","name":"p", - "created":"2026-01-02T03:04:05Z","updated":"2026-02-03T04:05:06Z"} - """); - - var project = await _apiClient.GetOrCreateProject("p"); - - Assert.Equal("2026-02-03T04:05:06Z", project.Updated); - Assert.NotNull(project.Created); - // `created` is typed in the spec, so it is normalized to round-trip ISO 8601 - // rather than echoed verbatim. - Assert.Equal( - DateTimeOffset.Parse("2026-01-02T03:04:05Z"), - DateTimeOffset.Parse(project.Created!)); - } - - [Fact] - public async Task GetOrCreateProjectAndOrgInfo_resolves_the_org_through_login() - { - _handler.Enqueue(HttpStatusCode.OK, ProjectJson); - _handler.Enqueue(HttpStatusCode.OK, - $$"""{"org_info":[{"id":"{{OrgId}}","name":"Test Org"}]}"""); - - var info = await _apiClient.GetOrCreateProjectAndOrgInfo(); - - Assert.Equal(ProjectId, info.Project.Id); - Assert.Equal(OrgId, info.OrgInfo.Id); - Assert.Equal("Test Org", info.OrgInfo.Name); - - // Login is absent from the OpenAPI spec, so it is issued directly. - Assert.Equal("/api/apikey/login", _handler.LastRequest?.RequestUri?.AbsolutePath); - } - - [Fact] - public async Task Http_error_surfaces_as_an_sdk_ApiException() - { - _handler.Enqueue(HttpStatusCode.BadRequest, "Bad request"); - - var exception = await Assert.ThrowsAsync(() => _apiClient.GetProject(ProjectId)); - - Assert.Equal(400, exception.StatusCode); - Assert.Contains("400", exception.Message); - // The server's body is the whole diagnostic value of an error; the generated - // client discards it unless asked to keep it. - Assert.Contains("Bad request", exception.Message); - } - - [Theory] - [InlineData("Bad request")] - [InlineData("\"Bad request\"")] - [InlineData("{\"error\":\"Bad request\"}")] - public async Task Http_error_reports_the_server_body_whatever_shape_it_takes(string body) - { - // The spec declares these error bodies as plain strings, but the API answers with - // bare text and with JSON objects too. None of those may come back empty. - _handler.Enqueue(HttpStatusCode.InternalServerError, body); - - var exception = await Assert.ThrowsAsync( - () => _apiClient.GetOrCreateProject("p")); - - Assert.Equal(500, exception.StatusCode); - Assert.Contains("Bad request", exception.Message); - } - - [Fact] - public async Task Non_uuid_identifier_reports_a_clear_error() - { - // The spec types identifiers as UUIDs, so the generated client does too. This is - // stricter than the hand-rolled client, which pasted any string into the URL. - var exception = await Assert.ThrowsAsync(() => _apiClient.GetProject("proj-123")); - - Assert.Contains("must be a UUID", exception.Message); - Assert.Contains("proj-123", exception.Message); - Assert.Null(_handler.LastRequest); - } - - [Fact] - public async Task Uses_api_key_from_the_braintrust_env_file() - { - var originalCwd = Directory.GetCurrentDirectory(); - var originalApiKey = Environment.GetEnvironmentVariable("BRAINTRUST_API_KEY"); - var tempDir = Directory.CreateTempSubdirectory("braintrust-default-api-client-env-").FullName; - - try - { - Environment.SetEnvironmentVariable("BRAINTRUST_API_KEY", null); - File.WriteAllText(Path.Combine(tempDir, ".env.braintrust"), "BRAINTRUST_API_KEY=file-api-key\n"); - Directory.SetCurrentDirectory(tempDir); - - using var handler = new StubHandler(); - var config = BraintrustConfig.Of( - ("BRAINTRUST_API_URL", "https://test-api.example.com"), - ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") - ); - using var apiClient = new DefaultBraintrustApiClient(config, handler); - handler.Enqueue(HttpStatusCode.OK, ProjectJson); - - await apiClient.GetOrCreateProject("test-project"); - - Assert.Equal("Bearer file-api-key", handler.LastRequest?.Headers.Authorization?.ToString()); - } - finally - { - Directory.SetCurrentDirectory(originalCwd); - Environment.SetEnvironmentVariable("BRAINTRUST_API_KEY", originalApiKey); - Directory.Delete(tempDir, recursive: true); - } - } - - [Fact] - public void Braintrust_resolves_to_the_generated_backed_client_by_default() - { - // Pins the migration: the SDK's own entry point must not fall back to the - // deprecated hand-rolled client. - var config = BraintrustConfig.Of( - ("BRAINTRUST_API_KEY", "test-api-key"), - ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project")); - - var braintrust = Braintrust.Of(config, autoManageOpenTelemetry: false); - - Assert.IsType(braintrust.ApiClient); - } - - [Fact] - public async Task Api_exposes_the_generated_client_wired_to_this_config() - { - // The generated client covers endpoints IBraintrustApiClient does not wrap, so it - // is public - but it has to inherit this instance's base URL and API key. - _handler.Enqueue(HttpStatusCode.OK, $$"""{"objects":[{{ProjectJson}}]}"""); - - var page = await _apiClient.Api.GetProjectAsync( - limit: 1, - starting_after: null, - ending_before: null, - ids: null, - project_name: "test-project", - org_name: null); - - var project = Assert.Single(page.Objects); - Assert.Equal(Guid.Parse(ProjectId), project.Id); - - Assert.Equal("test-api.example.com", _handler.LastRequest?.RequestUri?.Host); - Assert.Equal("/v1/project", _handler.LastRequest?.RequestUri?.AbsolutePath); - Assert.Contains("project_name=test-project", _handler.LastRequest?.RequestUri?.Query); - Assert.Equal("Bearer test-api-key", _handler.LastRequest?.Headers.Authorization?.ToString()); - } - - [Fact] - public async Task Api_surfaces_failures_as_the_generated_exception() - { - // Only the wrapper methods translate to Braintrust.Sdk.Api.ApiException; a direct - // generated call raises the generated one, and keeps the server's body. - _handler.Enqueue(HttpStatusCode.NotFound, "no such project"); - - var ex = await Assert.ThrowsAsync( - () => _apiClient.Api.GetProjectIdAsync(Guid.Parse(ProjectId))); - - Assert.Equal(404, ex.StatusCode); - Assert.Contains("no such project", ex.Response); - } - - private sealed class StubHandler : HttpMessageHandler - { - private readonly Queue<(HttpStatusCode Status, string Body)> _responses = new(); - - public HttpRequestMessage? LastRequest { get; private set; } - public string? LastRequestBody { get; private set; } - - public void Enqueue(HttpStatusCode status, string body) => _responses.Enqueue((status, body)); - - protected override async Task SendAsync( - HttpRequestMessage request, CancellationToken cancellationToken) - { - LastRequest = request; - LastRequestBody = request.Content is null - ? null - : await request.Content.ReadAsStringAsync(cancellationToken); - - if (_responses.Count == 0) - { - throw new InvalidOperationException("No response configured for test"); - } - - var (status, body) = _responses.Dequeue(); - return new HttpResponseMessage(status) - { - Content = new StringContent(body, Encoding.UTF8, "application/json"), - }; - } - } -} diff --git a/tests/Braintrust.Sdk.Tests/Api/DeprecatedApisTest.cs b/tests/Braintrust.Sdk.Tests/Api/DeprecatedApisTest.cs new file mode 100644 index 0000000..aa049f8 --- /dev/null +++ b/tests/Braintrust.Sdk.Tests/Api/DeprecatedApisTest.cs @@ -0,0 +1,80 @@ +using System.Net; +using Braintrust.Sdk.Api; +using Braintrust.Sdk.Config; +using Braintrust.Sdk.Git; + +#pragma warning disable CS0618 // Compatibility APIs are intentionally exercised here. + +namespace Braintrust.Sdk.Tests.Api; + +[Collection("BraintrustGlobals")] +public class DeprecatedApisTest +{ + private const string ProjectId = "3f2504e0-4f89-11d3-9a0c-0305e82c3301"; + private const string OrgId = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; + + private static BraintrustConfig Config() => BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-api-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com")); + + private static string ProjectJson => + $$"""{"id":"{{ProjectId}}","org_id":"{{OrgId}}","name":"test-project"}"""; + + [Fact] + public async Task Former_clients_delegate_to_the_openapi_client() + { + using var legacyHandler = new QueuedHttpHandler(); + legacyHandler.Enqueue(ProjectJson); + using var httpClient = new HttpClient(legacyHandler); + using var legacyClient = new BraintrustApiClient(Config(), httpClient); + + using var defaultHandler = new QueuedHttpHandler(); + defaultHandler.Enqueue(ProjectJson); + using var defaultClient = new DefaultBraintrustApiClient(Config(), defaultHandler); + + var legacyProject = await legacyClient.GetProject(ProjectId); + var defaultProject = await defaultClient.GetProject(ProjectId); + + Assert.Equal(ProjectId, legacyProject?.Id); + Assert.Equal(ProjectId, defaultProject?.Id); + Assert.NotNull(defaultClient.Api); + } + + [Fact] + public async Task Braintrust_ApiClient_remains_usable() + { + using var handler = new QueuedHttpHandler(); + handler.Enqueue(ProjectJson); + using var openApiClient = new BraintrustOpenApiClient(Config(), handler); + var braintrust = Braintrust.Of(Config(), openApiClient, autoManageOpenTelemetry: false); + + var project = await braintrust.ApiClient.GetProject(ProjectId); + + Assert.Equal(ProjectId, project?.Id); + } + + [Fact] + public void CreateExperimentRequest_supports_the_former_record_shape() + { + var repoInfo = new RepoInfo(Commit: "abc123"); + IReadOnlyList tags = ["nightly"]; + IReadOnlyDictionary metadata = + new Dictionary { ["run"] = 7 }; + + var request = new CreateExperimentRequest( + ProjectId, "experiment", "description", "base-id", repoInfo, tags, metadata); + + var (projectId, name, description, baseExperimentId, actualRepoInfo, actualTags, + actualMetadata) = request; + + Assert.Equal(ProjectId, projectId); + Assert.Equal("experiment", name); + Assert.Equal("description", description); + Assert.Equal("base-id", baseExperimentId); + Assert.Same(repoInfo, actualRepoInfo); + Assert.Same(tags, actualTags); + Assert.Same(metadata, actualMetadata); + } +} + +#pragma warning restore CS0618 diff --git a/tests/Braintrust.Sdk.Tests/Api/Internal/BtqlClientTest.cs b/tests/Braintrust.Sdk.Tests/Api/Internal/BtqlClientTest.cs index a74e0f9..3bd2a2c 100644 --- a/tests/Braintrust.Sdk.Tests/Api/Internal/BtqlClientTest.cs +++ b/tests/Braintrust.Sdk.Tests/Api/Internal/BtqlClientTest.cs @@ -1,7 +1,7 @@ using System.Net; using System.Text; using System.Text.Json; -using Braintrust.Sdk.Api.Internal; +using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; namespace Braintrust.Sdk.Tests.Api.Internal; @@ -23,62 +23,69 @@ private static HttpResponseMessage MakeJsonResponse(string json, HttpStatusCode }; } - private static string MakeBtqlResponse(string freshness, int rowCount = 1) + private static string MakeBtqlResponse(int rowCount = 1) { var rows = Enumerable.Range(0, rowCount) - .Select(i => $"{{\"span_id\":\"{i}\",\"span_attributes\":{{\"type\":\"task\"}}}}") + .Select(i => new { span_id = i.ToString(), span_attributes = new { type = "task" } }) .ToList(); - return $"{{\"data\":[{string.Join(",", rows)}],\"freshness\":\"{freshness}\"}}"; + return JsonSerializer.Serialize(new + { + data = rows, + freshness_state = new + { + last_processed_xact_id = "42", + last_considered_xact_id = "42", + }, + realtime_state = new { type = "on" }, + }); } [Fact] public async Task ReturnsDataWhenFreshnessIsComplete() { var handler = new MockHttpMessageHandler(); - handler.Enqueue(MakeJsonResponse(MakeBtqlResponse("complete", 2))); + handler.Enqueue(MakeJsonResponse(MakeBtqlResponse(2))); - var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://api.braintrust.dev") }; - var client = new BtqlClient(MakeConfig(), httpClient, noDelay: true); + using var client = new BraintrustOpenApiClient(MakeConfig(), handler, noBtqlDelay: true); - var result = await client.QuerySpansAsync("exp-id", "root-span-id"); + var result = await client.QuerySpansAsync("exp-id", "root-span-id", ["0", "1"]); Assert.Equal(2, result.Count); Assert.Equal(1, handler.RequestCount); } [Fact] - public async Task RetriesWhenFreshnessIsPartial() + public async Task RetriesUntilTheExpectedSpanAppears() { var handler = new MockHttpMessageHandler(); - handler.Enqueue(MakeJsonResponse(MakeBtqlResponse("partial", 1))); - handler.Enqueue(MakeJsonResponse(MakeBtqlResponse("partial", 1))); - handler.Enqueue(MakeJsonResponse(MakeBtqlResponse("complete", 3))); + handler.Enqueue(MakeJsonResponse(MakeBtqlResponse(1))); + handler.Enqueue(MakeJsonResponse(MakeBtqlResponse(1))); + handler.Enqueue(MakeJsonResponse(MakeBtqlResponse(3))); - var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://api.braintrust.dev") }; - var client = new BtqlClient(MakeConfig(), httpClient, noDelay: true); + using var client = new BraintrustOpenApiClient(MakeConfig(), handler, noBtqlDelay: true); - var result = await client.QuerySpansAsync("exp-id", "root-span-id"); + var result = await client.QuerySpansAsync("exp-id", "root-span-id", ["2"]); Assert.Equal(3, result.Count); Assert.Equal(3, handler.RequestCount); } [Fact] - public async Task ReturnsDataAfterMaxRetriesEvenIfNotComplete() + public async Task ThrowsAfterMaxRetriesWhenTheExpectedSpanIsMissing() { var handler = new MockHttpMessageHandler(); // Enqueue 8 partial responses (1 initial + 7 retries) for (int i = 0; i < 8; i++) { - handler.Enqueue(MakeJsonResponse(MakeBtqlResponse("partial", 1))); + handler.Enqueue(MakeJsonResponse(MakeBtqlResponse(1))); } - var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://api.braintrust.dev") }; - var client = new BtqlClient(MakeConfig(), httpClient, noDelay: true); + using var client = new BraintrustOpenApiClient(MakeConfig(), handler, noBtqlDelay: true); - var result = await client.QuerySpansAsync("exp-id", "root-span-id"); + var exception = await Assert.ThrowsAsync( + () => client.QuerySpansAsync("exp-id", "root-span-id", ["missing"])); - Assert.NotEmpty(result); + Assert.Contains("missing", exception.Message); Assert.Equal(8, handler.RequestCount); // 1 initial + 7 retries } @@ -88,16 +95,15 @@ public async Task ReturnsEmptyWhenNoDataAfterMaxRetries() var handler = new MockHttpMessageHandler(); for (int i = 0; i < 8; i++) { - handler.Enqueue(MakeJsonResponse("{\"data\":[],\"freshness\":\"partial\"}")); + handler.Enqueue(MakeJsonResponse(MakeBtqlResponse(rowCount: 0))); } - var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://api.braintrust.dev") }; - var client = new BtqlClient(MakeConfig(), httpClient, noDelay: true); + using var client = new BraintrustOpenApiClient(MakeConfig(), handler, noBtqlDelay: true); - var result = await client.QuerySpansAsync("exp-id", "root-span-id"); + var result = await client.QuerySpansAsync("exp-id", "root-span-id", []); Assert.Empty(result); - Assert.Equal(8, handler.RequestCount); + Assert.Equal(1, handler.RequestCount); } [Fact] @@ -114,13 +120,13 @@ public async Task UsesCorrectBackoffSchedule() // All partial responses so we hit all retries for (int i = 0; i < 8; i++) { - handler.Enqueue(MakeJsonResponse("{\"data\":[],\"freshness\":\"partial\"}")); + handler.Enqueue(MakeJsonResponse(MakeBtqlResponse(rowCount: 0))); } - var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://api.braintrust.dev") }; - var client = new BtqlClient(MakeConfig(), httpClient, delayFunc: captureDelay); + using var client = new BraintrustOpenApiClient(MakeConfig(), handler, btqlDelayFunc: captureDelay); - await client.QuerySpansAsync("exp-id", "root-span-id"); + await Assert.ThrowsAsync( + () => client.QuerySpansAsync("exp-id", "root-span-id", ["missing"])); // Expected delays: 1s, 2s, 4s, 8s, 8s, 8s, 8s (7 delays for 7 retries) Assert.Equal(7, delays.Count); @@ -135,13 +141,12 @@ public async Task EscapesSingleQuotesInIds() var handler = new MockHttpMessageHandler(async req => { capturedRequests.Add(await req.Content!.ReadAsStringAsync()); - return MakeJsonResponse(MakeBtqlResponse("complete", 1)); + return MakeJsonResponse(MakeBtqlResponse(1)); }); - var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://api.braintrust.dev") }; - var client = new BtqlClient(MakeConfig(), httpClient, noDelay: true); + using var client = new BraintrustOpenApiClient(MakeConfig(), handler, noBtqlDelay: true); - await client.QuerySpansAsync("exp'id", "root'span'id"); + await client.QuerySpansAsync("exp'id", "root'span'id", ["0"]); Assert.Single(capturedRequests); var body = JsonSerializer.Deserialize(capturedRequests[0]); @@ -159,13 +164,12 @@ public async Task SendsAuthorizationHeader() var handler = new MockHttpMessageHandler(req => { capturedAuth = req.Headers.Authorization?.ToString(); - return Task.FromResult(MakeJsonResponse(MakeBtqlResponse("complete", 1))); + return Task.FromResult(MakeJsonResponse(MakeBtqlResponse(1))); }); - var httpClient = new HttpClient(handler) { BaseAddress = new Uri("https://api.braintrust.dev") }; - var client = new BtqlClient(MakeConfig(), httpClient, noDelay: true); + using var client = new BraintrustOpenApiClient(MakeConfig(), handler, noBtqlDelay: true); - await client.QuerySpansAsync("exp-id", "root-span-id"); + await client.QuerySpansAsync("exp-id", "root-span-id", ["0"]); Assert.Equal("Bearer test-key", capturedAuth); } diff --git a/tests/Braintrust.Sdk.Tests/Braintrust.Sdk.Tests.csproj b/tests/Braintrust.Sdk.Tests/Braintrust.Sdk.Tests.csproj index f518116..0c674b6 100644 --- a/tests/Braintrust.Sdk.Tests/Braintrust.Sdk.Tests.csproj +++ b/tests/Braintrust.Sdk.Tests/Braintrust.Sdk.Tests.csproj @@ -27,7 +27,7 @@ diff --git a/tests/Braintrust.Sdk.Tests/BraintrustTest.cs b/tests/Braintrust.Sdk.Tests/BraintrustTest.cs index 91de382..d0a2793 100644 --- a/tests/Braintrust.Sdk.Tests/BraintrustTest.cs +++ b/tests/Braintrust.Sdk.Tests/BraintrustTest.cs @@ -1,3 +1,4 @@ +using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; namespace Braintrust.Sdk.Tests; @@ -161,4 +162,142 @@ public void ConfigCannotBeNull() { Assert.Throws(() => Braintrust.Of(null!)); } + + [Fact] + public async Task FetchDatasetAsyncReadsFromTheConfiguredProject() + { + const string datasetId = "b9356d7d-1a96-4f96-9d41-276e9ebd6afe"; + const string projectId = "3f2504e0-4f89-11d3-9a0c-0305e82c3301"; + const string orgId = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; + + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "my-project") + ); + + using var handler = new QueuedHttpHandler(); + handler.Enqueue($$""" + {"objects":[{"id":"{{projectId}}","org_id":"{{orgId}}","name":"my-project"}]} + """); + handler.Enqueue($$""" + {"objects":[{"id":"{{datasetId}}","project_id":"{{projectId}}","name":"food"}]} + """); + + using var apiClient = new BraintrustOpenApiClient(config, handler); + var braintrust = Braintrust.Of(config, apiClient); + + var dataset = await braintrust.FetchDatasetAsync("food"); + + Assert.Equal(datasetId, dataset.Id); + + // A read resolves the project by name without upserting it, and never needs its org. + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Post); + Assert.DoesNotContain(handler.Requests, r => r.Path.StartsWith("/v1/organization")); + + // Scoped by project id: a project name is only unique within an org. + var lookup = handler.Requests[^1]; + Assert.Equal("/v1/dataset", lookup.Path); + Assert.Contains("dataset_name=food", lookup.Query); + Assert.Contains($"project_id={projectId}", lookup.Query); + Assert.DoesNotContain("project_name", lookup.Query); + } + + [Fact] + public async Task FetchDatasetAsyncDoesNotCreateAMistypedProject() + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "typo-project") + ); + + using var handler = new QueuedHttpHandler(); + handler.Enqueue("""{"objects":[]}"""); + + using var apiClient = new BraintrustOpenApiClient(config, handler); + var braintrust = Braintrust.Of(config, apiClient); + + var error = await Assert.ThrowsAsync( + () => braintrust.FetchDatasetAsync("food")); + + Assert.Contains("typo-project", error.Message); + Assert.DoesNotContain(handler.Requests, r => r.Method == HttpMethod.Post); + } + + [Fact] + public async Task FetchDatasetAsyncForwardsBothConverters() + { + const string datasetId = "b9356d7d-1a96-4f96-9d41-276e9ebd6afe"; + const string projectId = "3f2504e0-4f89-11d3-9a0c-0305e82c3301"; + const string orgId = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; + + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "my-project") + ); + + using var handler = new QueuedHttpHandler(); + handler.Enqueue($$""" + {"objects":[{"id":"{{projectId}}","org_id":"{{orgId}}","name":"my-project"}]} + """); + handler.Enqueue($$""" + {"objects":[{"id":"{{datasetId}}","project_id":"{{projectId}}","name":"food"}]} + """); + handler.Enqueue($$""" + {"events":[{ + "id":"row-1","_xact_id":"1000","created":"2026-01-01T00:00:00Z", + "project_id":"{{projectId}}","dataset_id":"{{datasetId}}", + "input":"one","expected":"uno" + }]} + """); + + using var apiClient = new BraintrustOpenApiClient(config, handler); + var braintrust = Braintrust.Of(config, apiClient); + + // Pinned, so the read goes straight to the rows without a version lookup. + var dataset = await braintrust.FetchDatasetAsync( + "food", + version: "1000", + inputConverter: e => $"input:{e.GetString()}", + expectedConverter: e => $"expected:{e.GetString()}"); + + var cases = new List>(); + await foreach (var datasetCase in dataset.GetCasesAsync()) + { + cases.Add(datasetCase); + } + + var only = Assert.Single(cases); + Assert.Equal("input:one", only.Input); + Assert.Equal("expected:uno", only.Expected); + } + + [Fact] + public async Task GetProjectUriAsyncKeepsAnyPathPrefixInTheAppUrl() + { + const string projectId = "3f2504e0-4f89-11d3-9a0c-0305e82c3301"; + const string orgId = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; + + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com"), + ("BRAINTRUST_APP_URL", "https://proxy.example.com/braintrust"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "my project") + ); + + using var handler = new QueuedHttpHandler(); + handler.Enqueue($$"""{"objects":[{"id":"{{projectId}}","org_id":"{{orgId}}","name":"my project"}]}"""); + handler.Enqueue($$"""{"id":"{{orgId}}","name":"my org"}"""); + + using var apiClient = new BraintrustOpenApiClient(config, handler); + var braintrust = Braintrust.Of(config, apiClient); + + var uri = await braintrust.GetProjectUriAsync(); + + Assert.Equal( + "https://proxy.example.com/braintrust/app/my%20org/p/my%20project", + uri.AbsoluteUri); + } } diff --git a/tests/Braintrust.Sdk.Tests/Eval/BtqlTestData.cs b/tests/Braintrust.Sdk.Tests/Eval/BtqlTestData.cs new file mode 100644 index 0000000..31d16f6 --- /dev/null +++ b/tests/Braintrust.Sdk.Tests/Eval/BtqlTestData.cs @@ -0,0 +1,24 @@ +using System.Text.Json; + +namespace Braintrust.Sdk.Tests.Eval; + +internal static class BtqlTestData +{ + internal static IReadOnlyDictionary MakeSpan( + string type, + object? input = null, + object? output = null) + { + var span = new Dictionary + { + ["span_id"] = Guid.NewGuid().ToString(), + ["span_attributes"] = new { type }, + }; + if (input != null) span["input"] = input; + if (output != null) span["output"] = output; + + return JsonSerializer.SerializeToElement(span) + .EnumerateObject() + .ToDictionary(property => property.Name, property => property.Value.Clone()); + } +} diff --git a/tests/Braintrust.Sdk.Tests/Eval/ClassifierTest.cs b/tests/Braintrust.Sdk.Tests/Eval/ClassifierTest.cs index b790130..11f8bcf 100644 --- a/tests/Braintrust.Sdk.Tests/Eval/ClassifierTest.cs +++ b/tests/Braintrust.Sdk.Tests/Eval/ClassifierTest.cs @@ -121,13 +121,13 @@ public async Task FunctionClassifierAsyncList() public async Task EvalRequiresAtLeastScorersOrClassifiers() { var config = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "test-key")); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var ex = await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .BuildAsync()); @@ -142,12 +142,12 @@ public async Task EvalBuildsWithClassifiersOnly() ("BRAINTRUST_API_KEY", "test-key"), ("BRAINTRUST_APP_URL", "https://braintrust.dev"), ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project")); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var eval = await Eval.NewBuilder() .Name("test-eval") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("hello", "hi")) .TaskFunction(x => x) .Classifiers(new FunctionClassifier( @@ -440,11 +440,9 @@ public async Task TracedClassifierReceivesEvalTrace() { var spans = new[] { - MockBtqlClient.MakeSpan("llm", input: new { messages = new[] { new { role = "user", content = "hi" } } }, + BtqlTestData.MakeSpan("llm", input: new { messages = new[] { new { role = "user", content = "hi" } } }, output: new { choices = new[] { new { message = new { role = "assistant", content = "hello" } } } }) }; - var mockBtql = new MockBtqlClient(spans); - var capturedSpanCount = -1; var classifier = new TracedClassifier( "trace_inspector", @@ -459,11 +457,10 @@ public async Task TracedClassifierReceivesEvalTrace() cases: new[] { DatasetCase.Of("hello", "hi") }, taskFn: x => x, classifiers: new IClassifier[] { classifier }, - btqlClient: mockBtql); + btqlRows: spans); Assert.Single(rootSpans); Assert.Equal(1, capturedSpanCount); - Assert.Equal(1, mockBtql.QueryCount); } // ===================================================================== @@ -484,14 +481,13 @@ private static TaskResult MakeTaskResult(string input, string ou Func taskFn, IScorer[]? scorers = null, IClassifier[]? classifiers = null, - MockBtqlClient? btqlClient = null) + IReadOnlyList>? btqlRows = null) { var config = BraintrustConfig.Of( ("BRAINTRUST_API_KEY", "test-key"), ("BRAINTRUST_APP_URL", "https://braintrust.dev"), ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project")); - var mockClient = new MockBraintrustApiClient(); - btqlClient ??= new MockBtqlClient(); + using var mockClient = new StubBraintrustApi(config, btqlRows: btqlRows); var captured = new List(); using var listener = new ActivityListener @@ -505,8 +501,7 @@ private static TaskResult MakeTaskResult(string input, string ou var builder = Eval.NewBuilder() .Name("classifier-test") .Config(config) - .ApiClient(mockClient) - .BtqlClient(btqlClient) + .ApiClient(mockClient.Client) .Cases(cases) .TaskFunction(taskFn); diff --git a/tests/Braintrust.Sdk.Tests/Eval/DatasetBrainstoreImplTest.cs b/tests/Braintrust.Sdk.Tests/Eval/DatasetBrainstoreImplTest.cs new file mode 100644 index 0000000..43f9d0e --- /dev/null +++ b/tests/Braintrust.Sdk.Tests/Eval/DatasetBrainstoreImplTest.cs @@ -0,0 +1,514 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using Braintrust.Sdk.Api; +using Braintrust.Sdk.Config; +using Braintrust.Sdk.Eval; +using Generated = Braintrust.Sdk.Api.Generated; + +namespace Braintrust.Sdk.Tests.Eval; + +/// +/// Covers , which reads a dataset through the +/// generated OpenAPI client. Responses are raw JSON so these assert against the wire shape the +/// API actually returns, including BTQL version lookups which are absent from the spec. +/// +public class DatasetBrainstoreImplTest : IDisposable +{ + private const string DatasetId = "b9356d7d-1a96-4f96-9d41-276e9ebd6afe"; + private const string ProjectId = "3f2504e0-4f89-11d3-9a0c-0305e82c3301"; + private const string FetchPath = "/v1/dataset/" + DatasetId + "/fetch"; + + private readonly QueuedHttpHandler _handler = new(); + private readonly BraintrustOpenApiClient _apiClient; + + public DatasetBrainstoreImplTest() + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-api-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com") + ); + + _apiClient = new BraintrustOpenApiClient(config, _handler); + } + + public void Dispose() + { + _apiClient.Dispose(); + _handler.Dispose(); + } + + private DatasetBrainstoreImpl Dataset(string? version = null) + where TInput : notnull + where TOutput : notnull + => new(_apiClient, DatasetId, version); + + private static async Task>> Drain( + IDataset dataset) + where TInput : notnull + where TOutput : notnull + { + var cases = new List>(); + await foreach (var datasetCase in dataset.GetCasesAsync()) + { + cases.Add(datasetCase); + } + + return cases; + } + + [Fact] + public async Task Pages_until_the_cursor_runs_out() + { + _handler.Enqueue(Page("cursor-1", Row("row-1", "one", "1"), Row("row-2", "two", "2"))); + _handler.Enqueue(Page(null, Row("row-3", "three", "3"))); + + var cases = await Drain(Dataset(version: "1000")); + + Assert.Equal(["one", "two", "three"], cases.Select(c => c.Input)); + Assert.Equal(2, _handler.Requests.Count); + Assert.All(_handler.Requests, r => + { + Assert.Equal(HttpMethod.Post, r.Method); + Assert.Equal(FetchPath, r.Path); + Assert.Contains("\"version\":\"1000\"", r.Body); + }); + + // The cursor from page one is what asks for page two. + Assert.DoesNotContain("\"cursor\"", _handler.Requests[0].Body); + Assert.Contains("\"cursor\":\"cursor-1\"", _handler.Requests[1].Body); + } + + [Fact] + public async Task Older_versions_of_rows_on_later_pages_are_skipped() + { + _handler.Enqueue(Page("cursor-1", Row("row-1", "updated", "new", xactId: "1000"))); + _handler.Enqueue(Page(null, + Row("row-2", "two", "2", xactId: "900"), + Row("row-1", "original", "old", xactId: "800"))); + + var cases = await Drain(Dataset(version: "1000")); + + Assert.Equal(["updated", "two"], cases.Select(c => c.Input)); + Assert.Equal(["1000", "900"], cases.Select(c => c.Origin!.XactId)); + } + + [Fact] + public async Task An_empty_page_ends_the_enumeration_even_with_a_cursor() + { + _handler.Enqueue(Page("cursor-1", Row("row-1", "one", "1"))); + _handler.Enqueue(Page("cursor-2")); + + var cases = await Drain(Dataset(version: "1000")); + + Assert.Single(cases); + Assert.Equal(2, _handler.Requests.Count); + } + + [Fact] + public async Task Unpinned_datasets_read_at_the_latest_version() + { + GivenLatestVersion("1042", count: 1); + _handler.Enqueue(Page(null, Row("row-1", "one", "1"))); + + var dataset = Dataset(); + var cases = await Drain(dataset); + + Assert.Single(cases); + Assert.Equal("/btql", _handler.Requests[0].Path); + var query = JsonDocument.Parse(_handler.Requests[0].Body).RootElement.GetProperty("query").GetString(); + Assert.Contains($"dataset('{DatasetId}')", query); + Assert.Contains("max(_xact_id)", query); + Assert.Contains("\"version\":\"1042\"", _handler.Requests[1].Body); + + // Unpinned datasets resolve independently on every enumeration and retain no snapshot state. + Assert.Null(dataset.Version); + } + + [Fact] + public async Task Opening_a_snapshot_resolves_its_version_without_fetching_rows() + { + GivenLatestVersion("1042", count: 1); + _handler.Enqueue(Page(null, Row("row-1", "one", "1"))); + + var snapshot = await Dataset().OpenSnapshotAsync(); + + Assert.Equal("1042", snapshot.Version); + Assert.Equal("/btql", Assert.Single(_handler.Requests).Path); + + var cases = new List>(); + await foreach (var datasetCase in snapshot.Cases) + { + cases.Add(datasetCase); + } + + Assert.Single(cases); + Assert.Equal(FetchPath, _handler.Requests[1].Path); + Assert.Contains("\"version\":\"1042\"", _handler.Requests[1].Body); + } + + [Fact] + public async Task An_unpinned_dataset_resolves_again_for_each_enumeration() + { + GivenLatestVersion("1042", count: 1); + _handler.Enqueue(Page(null, Row("row-1", "one", "1"))); + + var dataset = Dataset(); + await Drain(dataset); + + GivenLatestVersion("1043", count: 1); + _handler.Enqueue(Page(null, Row("row-2", "two", "2", xactId: "1043"))); + await Drain(dataset); + + Assert.Equal(2, _handler.Requests.Count(request => request.Path == "/btql")); + Assert.Contains("\"version\":\"1042\"", _handler.Requests[1].Body); + Assert.Contains("\"version\":\"1043\"", _handler.Requests[3].Body); + Assert.Null(dataset.Version); + } + + [Fact] + public async Task A_pinned_version_skips_the_version_lookup() + { + _handler.Enqueue(Page(null, Row("row-1", "one", "1"))); + + var dataset = Dataset(version: "999"); + await Drain(dataset); + + Assert.DoesNotContain(_handler.Requests, request => request.Path == "/btql"); + Assert.Contains("\"version\":\"999\"", _handler.Requests[0].Body); + Assert.Equal("999", dataset.Version); + } + + [Fact] + public async Task An_empty_dataset_yields_nothing_and_fetches_no_pages() + { + GivenLatestVersion(version: null, count: 0); + + var cases = await Drain(Dataset()); + + Assert.Empty(cases); + Assert.Equal("/btql", Assert.Single(_handler.Requests).Path); + } + + [Fact] + public async Task Rows_are_deserialized_into_the_case_types() + { + _handler.Enqueue(Page(null, Row( + "row-1", + new { text = "why is the sky blue?", difficulty = 3 }, + new { text = "rayleigh scattering", difficulty = 3 }, + xactId: "1042", + tags: ["science", "easy"], + metadata: new { model = "gpt-4", reviewer = "ark" }))); + + var cases = await Drain(Dataset(version: "1042")); + + var only = Assert.Single(cases); + Assert.Equal(new Question("why is the sky blue?", 3), only.Input); + Assert.Equal(new Question("rayleigh scattering", 3), only.Expected); + Assert.Equal(["science", "easy"], only.Tags); + + // model is a declared property on the generated type, reviewer is extension data; both + // belong in the one bag. + Assert.Equal("gpt-4", only.Metadata["model"].ToString()); + Assert.Equal("ark", only.Metadata["reviewer"].ToString()); + } + + [Fact] + public async Task Cases_point_back_at_the_row_they_came_from() + { + _handler.Enqueue(Page(null, Row("row-1", "one", "1", xactId: "1042"))); + + var cases = await Drain(Dataset(version: "1042")); + + var origin = Assert.Single(cases).Origin; + Assert.NotNull(origin); + Assert.Equal("dataset", origin.ObjectType); + Assert.Equal(DatasetId, origin.ObjectId); + Assert.Equal("row-1", origin.Id); + Assert.Equal("1042", origin.XactId); + Assert.StartsWith("2026-01-01", origin.Created); + } + + [Fact] + public async Task A_row_with_no_expected_is_reported_with_its_id() + { + _handler.Enqueue(Page(null, Row("row-1", "one", expected: null))); + + var error = await Assert.ThrowsAsync( + () => Drain(Dataset(version: "1000"))); + + Assert.Contains("row-1", error.Message); + Assert.Contains("expected", error.Message); + } + + [Fact] + public async Task A_converter_reads_rows_the_default_deserializer_cannot() + { + _handler.Enqueue(Page(null, + Row("row-1", "one", expected: "uno"), + Row("row-2", "two", expected: null))); + + // The escape hatch for an optional 'expected': the converter sees the JSON null the + // missing field is normalized to, and decides what it means. + var dataset = Sdk.Eval.Dataset.FromId( + _apiClient, + DatasetId, + version: "1000", + expectedConverter: e => e.ValueKind == JsonValueKind.Null ? "" : e.GetString()!); + + var cases = await Drain(dataset); + + Assert.Equal(["one", "two"], cases.Select(c => c.Input)); + Assert.Equal(["uno", ""], cases.Select(c => c.Expected)); + } + + [Fact] + public async Task A_failed_fetch_surfaces_as_the_generated_exception() + { + _handler.Enqueue("""{"error":"nope"}""", HttpStatusCode.InternalServerError); + + var error = await Assert.ThrowsAnyAsync( + () => Drain(Dataset(version: "1000"))); + + Assert.Equal(500, error.StatusCode); + Assert.Contains("nope", error.Message); + } + + [Fact] + public async Task A_failed_name_lookup_surfaces_as_the_generated_exception() + { + _handler.Enqueue("""{"error":"nope"}""", HttpStatusCode.Forbidden); + + var error = await Assert.ThrowsAnyAsync( + () => Sdk.Eval.Dataset.FetchFromBraintrustAsync( + _apiClient, "my-project", "qa")); + + Assert.Equal(403, error.StatusCode); + } + + [Fact] + public async Task Names_resolve_to_ids() + { + _handler.Enqueue($$""" + {"objects":[{"id":"{{DatasetId}}","project_id":"{{ProjectId}}","name":"qa"}]} + """); + + var dataset = await Sdk.Eval.Dataset.FetchFromBraintrustAsync( + _apiClient, "my-project", "qa"); + + Assert.Equal(DatasetId, dataset.Id); + Assert.Equal("/v1/dataset", _handler.Requests[0].Path); + } + + [Fact] + public async Task A_name_that_matches_nothing_is_an_error() + { + _handler.Enqueue("""{"objects":[]}"""); + + var error = await Assert.ThrowsAsync( + () => Sdk.Eval.Dataset.FetchFromBraintrustAsync( + _apiClient, "my-project", "nope")); + + Assert.Contains("nope", error.Message); + Assert.Contains("my-project", error.Message); + } + + [Fact] + public async Task Enumerating_with_a_cancelled_token_makes_no_request() + { + _handler.Enqueue(Page(null, Row("row-1", "one", "uno"))); + + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + var dataset = Dataset(version: "1000"); + + // WithCancellation only reaches the fetch because GetCasesAsync hands back an iterator + // whose token parameter the compiler binds. + await Assert.ThrowsAnyAsync(async () => + { + await foreach (var _ in dataset.GetCasesAsync().WithCancellation(cts.Token)) + { + } + }); + + Assert.Empty(_handler.Requests); + } + + [Fact] + public async Task A_project_id_scopes_the_lookup_by_id() + { + _handler.Enqueue($$""" + {"objects":[{"id":"{{DatasetId}}","project_id":"{{ProjectId}}","name":"qa"}]} + """); + + var dataset = await Sdk.Eval.Dataset.FetchByProjectIdAsync( + _apiClient, ProjectId, "qa"); + + Assert.Equal(DatasetId, dataset.Id); + Assert.Contains($"project_id={ProjectId}", _handler.Requests[0].Query); + Assert.DoesNotContain("project_name", _handler.Requests[0].Query); + } + + [Fact] + public void An_invalid_dataset_id_is_rejected_before_any_request() + { + Assert.Throws( + () => Sdk.Eval.Dataset.FromId(_apiClient, "not-a-dataset-id")); + + Assert.Empty(_handler.Requests); + } + + [Fact] + public async Task A_name_that_matches_twice_is_an_error() + { + _handler.Enqueue($$""" + {"objects":[ + {"id":"{{DatasetId}}","project_id":"{{ProjectId}}","name":"qa"}, + {"id":"{{ProjectId}}","project_id":"{{ProjectId}}","name":"qa"} + ]} + """); + + var error = await Assert.ThrowsAsync( + () => Sdk.Eval.Dataset.FetchFromBraintrustAsync( + _apiClient, "my-project", "qa")); + + Assert.Contains("found 2", error.Message); + } + + [Fact] + public async Task An_input_converter_reads_rows_the_default_deserializer_cannot() + { + _handler.Enqueue(Page(null, + Row("row-1", new { text = "why is the sky blue?" }, expected: "rayleigh scattering"), + Row("row-2", new { text = "why is the sea salty?" }, expected: "runoff"))); + + // Rows store input as an object, so reading it as a string needs the converter; expected + // is left to the default deserializer to show the two sides are independent. + var dataset = Sdk.Eval.Dataset.FromId( + _apiClient, + DatasetId, + version: "1000", + inputConverter: e => e.GetProperty("text").GetString()!); + + var cases = await Drain(dataset); + + Assert.Equal(["why is the sky blue?", "why is the sea salty?"], cases.Select(c => c.Input)); + Assert.Equal(["rayleigh scattering", "runoff"], cases.Select(c => c.Expected)); + } + + [Fact] + public async Task Each_converter_reads_its_own_field() + { + _handler.Enqueue(Page(null, Row("row-1", "one", "uno"))); + + // Tagging the two apart catches converters wired to the wrong field. + var dataset = Sdk.Eval.Dataset.FromId( + _apiClient, + DatasetId, + version: "1000", + inputConverter: e => $"input:{e.GetString()}", + expectedConverter: e => $"expected:{e.GetString()}"); + + var only = Assert.Single(await Drain(dataset)); + + Assert.Equal("input:one", only.Input); + Assert.Equal("expected:uno", only.Expected); + } + + [Fact] + public async Task Fetching_by_name_forwards_both_converters() + { + _handler.Enqueue($$""" + {"objects":[{"id":"{{DatasetId}}","project_id":"{{ProjectId}}","name":"qa"}]} + """); + _handler.Enqueue(Page(null, Row("row-1", "one", "uno"))); + + var dataset = await Sdk.Eval.Dataset.FetchFromBraintrustAsync( + _apiClient, + "my-project", + "qa", + version: "1000", + inputConverter: e => $"input:{e.GetString()}", + expectedConverter: e => $"expected:{e.GetString()}"); + + var only = Assert.Single(await Drain(dataset)); + + Assert.Equal("input:one", only.Input); + Assert.Equal("expected:uno", only.Expected); + } + + [Fact] + public async Task Fetching_by_project_id_forwards_both_converters() + { + _handler.Enqueue($$""" + {"objects":[{"id":"{{DatasetId}}","project_id":"{{ProjectId}}","name":"qa"}]} + """); + _handler.Enqueue(Page(null, Row("row-1", "one", "uno"))); + + var dataset = await Sdk.Eval.Dataset.FetchByProjectIdAsync( + _apiClient, + ProjectId, + "qa", + version: "1000", + inputConverter: e => $"input:{e.GetString()}", + expectedConverter: e => $"expected:{e.GetString()}"); + + var only = Assert.Single(await Drain(dataset)); + + Assert.Equal("input:one", only.Input); + Assert.Equal("expected:uno", only.Expected); + } + + [Fact] + public async Task A_row_the_case_types_cannot_hold_is_reported_with_its_id() + { + _handler.Enqueue(Page(null, Row("row-1", new { text = "one" }, "uno"))); + + // The common mistake: an eval typed against a dataset of objects. The + // deserializer's own error names neither the row nor the field, so the wrapper adds both. + var error = await Assert.ThrowsAsync( + () => Drain(Dataset(version: "1000"))); + + Assert.Contains("row-1", error.Message); + Assert.Contains("input", error.Message); + Assert.Contains(nameof(String), error.Message); + Assert.IsType(error.InnerException); + } + + private record Question(string Text, int Difficulty); + + private void GivenLatestVersion(string? version, int count) => + _handler.Enqueue(JsonSerializer.Serialize(new + { + data = new[] { new { version, count } }, + freshness = "complete", + })); + + private static string Row( + string id, + object? input, + object? expected, + string xactId = "1000", + string[]? tags = null, + object? metadata = null) + => JsonSerializer.Serialize(new Dictionary + { + ["id"] = id, + ["_xact_id"] = xactId, + ["created"] = "2026-01-01T00:00:00Z", + ["project_id"] = ProjectId, + ["dataset_id"] = DatasetId, + ["input"] = input, + ["expected"] = expected, + ["tags"] = tags, + ["metadata"] = metadata, + }); + + private static string Page(string? cursor, params string[] rows) + { + var events = $"\"events\":[{string.Join(",", rows)}]"; + return cursor == null ? $"{{{events}}}" : $"{{{events},\"cursor\":\"{cursor}\"}}"; + } +} diff --git a/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs b/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs index cdea0e2..7b04cad 100644 --- a/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs +++ b/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs @@ -1,8 +1,11 @@ using System.Diagnostics; +using System.Net; using System.Text.Json; +using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; using Braintrust.Sdk.Eval; using Braintrust.Sdk.Git; +using Generated = Braintrust.Sdk.Api.Generated; namespace Braintrust.Sdk.Tests.Eval; @@ -41,7 +44,7 @@ public async Task BasicEvalBuildsAndRuns() ); // Create a mock API client that doesn't make real API calls - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var cases = new DatasetCase[] { @@ -62,7 +65,7 @@ public async Task BasicEvalBuildsAndRuns() var eval = await Eval.NewBuilder() .Name("test-eval") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(cases) .TaskFunction(food => "fruit") .Scorers( @@ -84,7 +87,7 @@ public async Task BasicEvalBuildsAndRuns() Assert.All(rootSpans, span => { Assert.Equal(ActivityStatusCode.Unset, span.Status); - Assert.Equal("experiment_id:test-experiment-id", span.GetTagItem("braintrust.parent")); + Assert.Equal($"experiment_id:{StubBraintrustApi.ExperimentId}", span.GetTagItem("braintrust.parent")); Assert.Equal("eval", GetSpanType(span)); Assert.NotNull(span.GetTagItem("braintrust.input_json")); Assert.NotNull(span.GetTagItem("braintrust.expected_json")); @@ -121,13 +124,13 @@ public async Task BasicEvalBuildsAndRuns() public async Task EvalRequiresAtLeastOneScorer() { var config = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "test-key")); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .BuildAsync()); @@ -137,13 +140,13 @@ await Assert.ThrowsAsync(() => public async Task EvalRequiresDataset() { var config = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "test-key")); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .TaskFunction(x => x) .Scorers(new FunctionScorer("test", (_, _) => 1.0)) .BuildAsync()); @@ -153,18 +156,43 @@ await Assert.ThrowsAsync(() => public async Task EvalRequiresTask() { var config = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "test-key")); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .Scorers(new FunctionScorer("test", (_, _) => 1.0)) .BuildAsync()); } + [Fact] + public async Task DisposingAnEvalLeavesACallerSuppliedClientOpen() + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") + ); + using var mockClient = new StubBraintrustApi(config); + + var eval = await Eval.NewBuilder() + .Name("test-eval") + .Config(config) + .ApiClient(mockClient.Client) + .Cases(DatasetCase.Of("input", "expected")) + .TaskFunction(x => x) + .Scorers(new FunctionScorer("test", (_, _) => 1.0)) + .BuildAsync(); + + eval.Dispose(); + + // Still usable: the eval only owns the clients it opened itself. + var project = await mockClient.Api.GetProjectIdAsync(Guid.Parse(StubBraintrustApi.ProjectId)); + Assert.Equal(StubBraintrustApi.ProjectId, project.Id.ToString()); + } + [Fact] public void DatasetCaseOfCreatesWithEmptyTagsAndMetadata() { @@ -217,7 +245,7 @@ public async Task EvalWithTagsAndMetadataBuildsAndRuns() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var tags = new List { "test-tag", "fruit-test" }; var metadata = new Dictionary @@ -236,7 +264,7 @@ public async Task EvalWithTagsAndMetadataBuildsAndRuns() var eval = await Eval.NewBuilder() .Name("test-eval-with-tags") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(cases) .TaskFunction(food => "fruit") .Scorers( @@ -261,7 +289,7 @@ public async Task EvalWithExperimentLevelTagsAndMetadataBuildsAndRuns() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var experimentTags = new[] { "experiment-tag", "dotnet-sdk", "v1" }; var experimentMetadata = new Dictionary @@ -281,7 +309,7 @@ public async Task EvalWithExperimentLevelTagsAndMetadataBuildsAndRuns() var eval = await Eval.NewBuilder() .Name("test-eval-with-experiment-metadata") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(cases) .TaskFunction(food => "fruit") .Tags(experimentTags) @@ -304,7 +332,7 @@ public async Task EvalWithExperimentLevelTagsAndMetadataBuildsAndRuns() Assert.Equal(3, lastRequest.Tags.Count); Assert.Contains("experiment-tag", lastRequest.Tags); Assert.NotNull(lastRequest.Metadata); - Assert.Equal("gpt-4o-mini", lastRequest.Metadata["model"]); + Assert.Equal("gpt-4o-mini", lastRequest.Metadata["model"].ToString()); } [Fact] @@ -317,14 +345,7 @@ public async Task TracedScorerReceivesEvalTrace() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); - - var spans = new List> - { - MockBtqlClient.MakeSpan("task"), - MockBtqlClient.MakeSpan("llm") - }; - var mockBtqlClient = new MockBtqlClient(spans); + using var mockClient = new StubBraintrustApi(config); EvalTrace? receivedTrace = null; var tracedScorer = new TestTracedScorer("traced_scorer", (taskResult, trace) => @@ -336,8 +357,7 @@ public async Task TracedScorerReceivesEvalTrace() var eval = await Eval.NewBuilder() .Name("test-eval-traced") .Config(config) - .ApiClient(mockClient) - .BtqlClient(mockBtqlClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(tracedScorer) @@ -351,22 +371,84 @@ public async Task TracedScorerReceivesEvalTrace() } [Fact] - public async Task TracedScorerCanAccessSpansViaTrace() + public async Task An_api_failure_surfaces_as_the_generated_exception() { var config = BraintrustConfig.Of( ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", "http://localhost:1"), ("BRAINTRUST_APP_URL", "https://braintrust.dev"), ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var handler = new QueuedHttpHandler(); + handler.Enqueue("""{"error":"boom"}""", HttpStatusCode.InternalServerError); + using var apiClient = new BraintrustOpenApiClient(config, handler); + + var error = await Assert.ThrowsAnyAsync(() => Eval.NewBuilder() + .Name("test-eval-failure") + .Config(config) + .ApiClient(apiClient) + .Cases(DatasetCase.Of("input", "expected")) + .TaskFunction(x => x) + .Scorers(new FunctionScorer("noop", (_, _) => 1.0)) + .BuildAsync()); + + Assert.Equal(500, error.StatusCode); + Assert.Contains("boom", error.Message); + } + + [Fact] + public async Task The_openapi_client_uses_its_transport_for_btql() + { + // Nothing listens on port 1: a separate BTQL transport would fail to connect instead + // of reaching the handler owned by this client. + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", "http://localhost:1"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") + ); + + using var mockClient = new StubBraintrustApi(config); + + IReadOnlyList>? spans = null; + var tracedScorer = new TestTracedScorer("span_checker", async (taskResult, trace) => + { + spans = await trace.GetSpansAsync(); + return [new Score("span_checker", 1.0)]; + }); + + using var eval = await Eval.NewBuilder() + .Name("test-eval-btql-transport") + .Config(config) + .ApiClient(mockClient.Client) + .Cases(DatasetCase.Of("input", "expected")) + .TaskFunction(x => x) + .Scorers(tracedScorer) + .BuildAsync(); + + await eval.RunAsync(); + + Assert.Contains("/btql", mockClient.Paths); + Assert.NotNull(spans); + Assert.Single(spans); + } + + [Fact] + public async Task TracedScorerCanAccessSpansViaTrace() + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") + ); var spans = new List> { - MockBtqlClient.MakeSpan("task"), - MockBtqlClient.MakeSpan("llm") + BtqlTestData.MakeSpan("task"), + BtqlTestData.MakeSpan("llm") }; - var mockBtqlClient = new MockBtqlClient(spans); + using var mockClient = new StubBraintrustApi(config, btqlRows: spans); IReadOnlyList>? receivedSpans = null; var tracedScorer = new TestTracedScorer("span_checker", async (taskResult, trace) => @@ -378,8 +460,7 @@ public async Task TracedScorerCanAccessSpansViaTrace() var eval = await Eval.NewBuilder() .Name("test-eval-spans") .Config(config) - .ApiClient(mockClient) - .BtqlClient(mockBtqlClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(tracedScorer) @@ -389,8 +470,7 @@ public async Task TracedScorerCanAccessSpansViaTrace() Assert.NotNull(receivedSpans); Assert.Equal(2, receivedSpans.Count); - // BtqlClient was queried exactly once for this eval case - Assert.Equal(1, mockBtqlClient.QueryCount); + Assert.Equal(1, mockClient.BtqlQueryCount); } [Fact] @@ -402,8 +482,7 @@ public async Task TracedScorerSpansAreLazilyFetched() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); - var mockBtqlClient = new MockBtqlClient(); + using var mockClient = new StubBraintrustApi(config); // Scorer that does NOT access the trace var tracedScorer = new TestTracedScorer("no_access", (taskResult, trace) => @@ -415,8 +494,7 @@ public async Task TracedScorerSpansAreLazilyFetched() var eval = await Eval.NewBuilder() .Name("test-eval-lazy") .Config(config) - .ApiClient(mockClient) - .BtqlClient(mockBtqlClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(tracedScorer) @@ -425,7 +503,7 @@ public async Task TracedScorerSpansAreLazilyFetched() await eval.RunAsync(); // BTQL was NOT queried because scorer never accessed the trace - Assert.Equal(0, mockBtqlClient.QueryCount); + Assert.Equal(0, mockClient.BtqlQueryCount); } [Fact] @@ -437,8 +515,7 @@ public async Task NonTracedScorerStillWorksAfterTracedScorerAdded() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); - var mockBtqlClient = new MockBtqlClient(); + using var mockClient = new StubBraintrustApi(config); var basicScorer = new FunctionScorer("basic", (_, _) => 1.0); var tracedScorer = new TestTracedScorer("traced", (taskResult, trace) => @@ -447,8 +524,7 @@ public async Task NonTracedScorerStillWorksAfterTracedScorerAdded() var eval = await Eval.NewBuilder() .Name("test-eval-mixed-scorers") .Config(config) - .ApiClient(mockClient) - .BtqlClient(mockBtqlClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(basicScorer, tracedScorer) @@ -460,7 +536,7 @@ public async Task NonTracedScorerStillWorksAfterTracedScorerAdded() } [Fact] - public async Task BasicScorerContinuesToWorkWithoutBtqlClient() + public async Task BasicScorerDoesNotQueryBtql() { var config = BraintrustConfig.Of( ("BRAINTRUST_API_KEY", "test-key"), @@ -468,13 +544,12 @@ public async Task BasicScorerContinuesToWorkWithoutBtqlClient() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); - // No traced scorers: BtqlClient is created but never queried var eval = await Eval.NewBuilder() .Name("test-eval-basic-only") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(new FunctionScorer("basic", (_, _) => 1.0)) @@ -482,6 +557,7 @@ public async Task BasicScorerContinuesToWorkWithoutBtqlClient() var result = await eval.RunAsync(); Assert.NotNull(result.ExperimentUrl); + Assert.Equal(0, mockClient.BtqlQueryCount); } [Fact] @@ -584,7 +660,7 @@ public async Task EvalWithMaxConcurrencyBuildsAndRuns() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var cases = new DatasetCase[] { @@ -597,7 +673,7 @@ public async Task EvalWithMaxConcurrencyBuildsAndRuns() var eval = await Eval.NewBuilder() .Name("test-eval-with-concurrency") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(cases) .TaskFunction(food => "fruit") .MaxConcurrency(2) @@ -612,23 +688,219 @@ public async Task EvalWithMaxConcurrencyBuildsAndRuns() Assert.Contains("test-eval-with-concurrency", result.ExperimentUrl); } + [Theory] + [InlineData(2)] + [InlineData(null)] + public async Task DatasetFailureWaitsForStartedCases(int? maxConcurrency) + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") + ); + using var mockClient = new StubBraintrustApi(config); + + var releaseTasks = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var firstTaskStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var started = 0; + var completed = 0; + + static async IAsyncEnumerable> FailingCases() + { + yield return new("one", "one"); + yield return new("two", "two"); + await Task.Yield(); + throw new InvalidOperationException("dataset failed"); + } + + var eval = await Eval.NewBuilder() + .Name("dataset-failure") + .Config(config) + .ApiClient(mockClient.Client) + .Dataset(new TestDataset(FailingCases)) + .TaskFunction(async input => + { + Interlocked.Increment(ref started); + firstTaskStarted.TrySetResult(); + await releaseTasks.Task; + Interlocked.Increment(ref completed); + return input; + }) + .MaxConcurrency(maxConcurrency) + .Scorers(new FunctionScorer("match", (expected, actual) => expected == actual ? 1.0 : 0.0)) + .BuildAsync(); + + var run = eval.RunAsync(); + await firstTaskStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + Assert.False(run.IsCompleted); + + releaseTasks.TrySetResult(); + var error = await Assert.ThrowsAsync(() => run); + Assert.Equal("dataset failed", error.Message); + Assert.Equal(started, completed); + } + + [Fact] + public async Task MaxConcurrencyAppliesBackpressureToDatasetEnumeration() + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") + ); + using var mockClient = new StubBraintrustApi(config); + + var releaseTasks = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var bothTasksStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var queueFilled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var enumerated = 0; + var started = 0; + + async IAsyncEnumerable> Cases() + { + for (var i = 0; i < 100; i++) + { + if (Interlocked.Increment(ref enumerated) == 4) + { + queueFilled.TrySetResult(); + } + yield return new(i.ToString(), i.ToString()); + await Task.Yield(); + } + } + + var eval = await Eval.NewBuilder() + .Name("bounded-dataset") + .Config(config) + .ApiClient(mockClient.Client) + .Dataset(new TestDataset(Cases)) + .TaskFunction(async input => + { + if (Interlocked.Increment(ref started) == 2) + { + bothTasksStarted.TrySetResult(); + } + await releaseTasks.Task; + return input; + }) + .MaxConcurrency(2) + .Scorers(new FunctionScorer("match", (expected, actual) => expected == actual ? 1.0 : 0.0)) + .BuildAsync(); + + var run = eval.RunAsync(); + await bothTasksStarted.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await queueFilled.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // Two cases are active and at most two more can be waiting in the bounded queue. + Assert.Equal(4, Volatile.Read(ref enumerated)); + + releaseTasks.TrySetResult(); + await run; + Assert.Equal(100, enumerated); + } + + [Theory] + [InlineData(1)] + [InlineData(null)] + public async Task CaseFailureStopsDatasetEnumerationAndQueuedWork(int? maxConcurrency) + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") + ); + using var mockClient = new StubBraintrustApi(config); + + var enumerated = 0; + var invoked = 0; + + async IAsyncEnumerable> Cases() + { + for (var i = 0; i < 100; i++) + { + Interlocked.Increment(ref enumerated); + yield return new(i.ToString(), i.ToString()); + await Task.Yield(); + } + } + + var eval = await Eval.NewBuilder() + .Name("failed-case-stops-producer") + .Config(config) + .ApiClient(mockClient.Client) + .Dataset(new TestDataset(Cases)) + .TaskFunction(input => + { + Interlocked.Increment(ref invoked); + return input; + }) + .MaxConcurrency(maxConcurrency) + .Scorers(new FunctionScorer("invalid", (_, _) => 2.0)) + .BuildAsync(); + + await Assert.ThrowsAsync(() => eval.RunAsync()); + + Assert.Equal(1, invoked); + Assert.InRange(enumerated, 1, 2); + } + + [Fact] + public async Task UnsampledTaskActivityDoesNotFailEvaluation() + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") + ); + using var mockClient = new StubBraintrustApi(config); + using var activitySource = new ActivitySource("eval-selective-sampling-test"); + using var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == activitySource.Name, + Sample = (ref ActivityCreationOptions options) => + options.Name == "eval" + ? ActivitySamplingResult.AllDataAndRecorded + : ActivitySamplingResult.None, + }; + ActivitySource.AddActivityListener(listener); + + var invoked = false; + var eval = await Eval.NewBuilder() + .Name("unsampled-task") + .Config(config) + .ApiClient(mockClient.Client) + .ActivitySource(activitySource) + .Cases(DatasetCase.Of("input", "input")) + .TaskFunction(input => + { + invoked = true; + return input; + }) + .Scorers(new FunctionScorer("match", (expected, actual) => expected == actual ? 1.0 : 0.0)) + .BuildAsync(); + + await eval.RunAsync(); + + Assert.True(invoked); + } + [Fact] public async Task EvalAutoCollectsRepoInfo() { // When running in a git repo (which the test suite is), repo_info should be - // auto-populated on the CreateExperimentRequest. + // auto-populated on the generated experiment request. var config = BraintrustConfig.Of( ("BRAINTRUST_API_KEY", "test-key"), ("BRAINTRUST_APP_URL", "https://braintrust.dev"), ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var eval = await Eval.NewBuilder() .Name("test-eval-repo-info") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(new FunctionScorer("match", (expected, actual) => expected == actual ? 1.0 : 0.0)) @@ -638,9 +910,9 @@ public async Task EvalAutoCollectsRepoInfo() var lastRequest = mockClient.LastCreateExperimentRequest; Assert.NotNull(lastRequest); - Assert.NotNull(lastRequest.RepoInfo); - Assert.NotNull(lastRequest.RepoInfo.Commit); - Assert.Matches("^[0-9a-f]{40}$", lastRequest.RepoInfo.Commit); + Assert.NotNull(lastRequest.Repo_info); + Assert.NotNull(lastRequest.Repo_info.Commit); + Assert.Matches("^[0-9a-f]{40}$", lastRequest.Repo_info.Commit); } [Fact] @@ -653,12 +925,12 @@ public async Task EvalExplicitNullRepoInfoDisablesCollection() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var eval = await Eval.NewBuilder() .Name("test-eval-no-repo-info") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(new FunctionScorer("match", (_, _) => 1.0)) @@ -669,7 +941,7 @@ public async Task EvalExplicitNullRepoInfoDisablesCollection() var lastRequest = mockClient.LastCreateExperimentRequest; Assert.NotNull(lastRequest); - Assert.Null(lastRequest.RepoInfo); + Assert.Null(lastRequest.Repo_info); } [Fact] @@ -681,7 +953,7 @@ public async Task EvalWithExplicitRepoInfoUsesProvidedValue() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var customRepoInfo = new RepoInfo( Commit: "abc123def456", Branch: "feature/test", @@ -691,7 +963,7 @@ public async Task EvalWithExplicitRepoInfoUsesProvidedValue() var eval = await Eval.NewBuilder() .Name("test-eval-custom-repo-info") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(new FunctionScorer("match", (_, _) => 1.0)) @@ -702,10 +974,10 @@ public async Task EvalWithExplicitRepoInfoUsesProvidedValue() var lastRequest = mockClient.LastCreateExperimentRequest; Assert.NotNull(lastRequest); - Assert.NotNull(lastRequest.RepoInfo); - Assert.Equal("abc123def456", lastRequest.RepoInfo.Commit); - Assert.Equal("feature/test", lastRequest.RepoInfo.Branch); - Assert.Equal(false, lastRequest.RepoInfo.Dirty); + Assert.NotNull(lastRequest.Repo_info); + Assert.Equal("abc123def456", lastRequest.Repo_info.Commit); + Assert.Equal("feature/test", lastRequest.Repo_info.Branch); + Assert.Equal(false, lastRequest.Repo_info.Dirty); } [Fact] @@ -717,12 +989,12 @@ public async Task EvalWithGitMetadataSettingsNoneOmitsRepoInfo() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var eval = await Eval.NewBuilder() .Name("test-eval-no-git") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(new FunctionScorer("match", (_, _) => 1.0)) @@ -733,7 +1005,7 @@ public async Task EvalWithGitMetadataSettingsNoneOmitsRepoInfo() var lastRequest = mockClient.LastCreateExperimentRequest; Assert.NotNull(lastRequest); - Assert.Null(lastRequest.RepoInfo); + Assert.Null(lastRequest.Repo_info); } // ------------------------------------------------------------------------- @@ -750,7 +1022,7 @@ public async Task TaskThrows_EvalCompletesAndScorerReceivesFallbackScore() ("BRAINTRUST_APP_URL", "https://braintrust.dev"), ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var capturedActivities = new List(); using var listener = new ActivityListener @@ -764,7 +1036,7 @@ public async Task TaskThrows_EvalCompletesAndScorerReceivesFallbackScore() var eval = await Eval.NewBuilder() .Name("test-task-throws") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction((Func)(_ => throw new InvalidOperationException("task boom"))) .Scorers(new FunctionScorer("my_scorer", (_, _) => 1.0)) @@ -807,7 +1079,7 @@ public async Task TaskThrows_MultipleScorers_AllReceiveFallbackScores() ("BRAINTRUST_APP_URL", "https://braintrust.dev"), ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var capturedActivities = new List(); using var listener = new ActivityListener @@ -821,7 +1093,7 @@ public async Task TaskThrows_MultipleScorers_AllReceiveFallbackScores() var eval = await Eval.NewBuilder() .Name("test-task-throws-multi") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction((Func)(_ => throw new InvalidOperationException("task boom"))) .Scorers( @@ -856,7 +1128,7 @@ public async Task ScorerThrows_OtherScorersStillRun_FailingGetsFallbackScore() ("BRAINTRUST_APP_URL", "https://braintrust.dev"), ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var capturedActivities = new List(); using var listener = new ActivityListener @@ -870,7 +1142,7 @@ public async Task ScorerThrows_OtherScorersStillRun_FailingGetsFallbackScore() var eval = await Eval.NewBuilder() .Name("test-scorer-throws") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers( @@ -917,7 +1189,7 @@ public async Task ScoreForScorerException_Override_ReturnsCustomFallbackScore() ("BRAINTRUST_APP_URL", "https://braintrust.dev"), ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var capturedActivities = new List(); using var listener = new ActivityListener @@ -931,7 +1203,7 @@ public async Task ScoreForScorerException_Override_ReturnsCustomFallbackScore() var eval = await Eval.NewBuilder() .Name("test-custom-fallback") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) .Scorers(new CustomFallbackScorer("custom_scorer", fallbackValue: 0.5)) @@ -957,7 +1229,7 @@ public async Task ScoreForTaskException_Override_ReturnsCustomFallbackScore() ("BRAINTRUST_APP_URL", "https://braintrust.dev"), ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - var mockClient = new MockBraintrustApiClient(); + using var mockClient = new StubBraintrustApi(config); var capturedActivities = new List(); using var listener = new ActivityListener @@ -971,7 +1243,7 @@ public async Task ScoreForTaskException_Override_ReturnsCustomFallbackScore() var eval = await Eval.NewBuilder() .Name("test-custom-task-exception-fallback") .Config(config) - .ApiClient(mockClient) + .ApiClient(mockClient.Client) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction((Func)(_ => throw new InvalidOperationException("task boom"))) .Scorers(new CustomFallbackScorer("custom_scorer", fallbackValue: 0.75)) @@ -995,13 +1267,12 @@ public async Task ExperimentUrlEscapesNamesWithSpaces() ("BRAINTRUST_DEFAULT_PROJECT_NAME", "my project") ); - var mockClient = new MockBraintrustApiClient(orgName: "Braintrust SDKs", projectName: "my project"); + var mockClient = new StubBraintrustApi(config, orgName: "Braintrust SDKs"); var eval = await Eval.NewBuilder() .Name("my eval") .Config(config) - .ApiClient(mockClient) - .BtqlClient(new MockBtqlClient()) + .ApiClient(mockClient.Client) .Cases(new DatasetCase("strawberry", "fruit")) .TaskFunction(food => "fruit") .Scorers(new FunctionScorer("exact", (expected, actual) => expected == actual ? 1.0 : 0.0)) @@ -1014,10 +1285,111 @@ public async Task ExperimentUrlEscapesNamesWithSpaces() result.ExperimentUrl); } + [Fact] + public async Task RemoteDatasetLinksTheExperimentAndEveryRow() + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") + ); + + const string datasetId = "b9356d7d-1a96-4f96-9d41-276e9ebd6afe"; + + using var handler = new QueuedHttpHandler(); + handler.Enqueue("""{"data":[{"version":"1042","count":1}],"freshness":"complete"}"""); + handler.Enqueue($$""" + {"events":[{ + "id":"row-1", + "_xact_id":"1042", + "created":"2026-01-01T00:00:00Z", + "project_id":"3f2504e0-4f89-11d3-9a0c-0305e82c3301", + "dataset_id":"{{datasetId}}", + "input":"strawberry", + "expected":"fruit" + }]} + """); + + using var apiClient = new BraintrustOpenApiClient(config, handler); + var dataset = new DatasetBrainstoreImpl(apiClient, datasetId); + + using var mockClient = new StubBraintrustApi(config); + var capturedActivities = new List(); + using var listener = new ActivityListener + { + ShouldListenTo = source => source.Name == "braintrust-dotnet", + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStopped = capturedActivities.Add + }; + ActivitySource.AddActivityListener(listener); + + var eval = await Eval.NewBuilder() + .Name("dataset-eval") + .Config(config) + .ApiClient(mockClient.Client) + .Dataset(dataset) + .TaskFunction(food => "fruit") + .Scorers(new FunctionScorer("exact", (expected, actual) => expected == actual ? 1.0 : 0.0)) + .BuildAsync(); + + await eval.RunAsync(); + + // The experiment points at the dataset and the exact version the run read. + Assert.Equal(Guid.Parse(datasetId), mockClient.LastCreateExperimentRequest?.Dataset_id); + Assert.Equal("1042", mockClient.LastCreateExperimentRequest?.Dataset_version); + + // ...and each row points back at the record it came from. + var rootSpan = Assert.Single(capturedActivities.Where(a => a.DisplayName == "eval")); + var origin = JsonSerializer.Deserialize( + (string)rootSpan.GetTagItem("braintrust.origin")!); + Assert.Equal("dataset", origin.GetProperty("object_type").GetString()); + Assert.Equal(datasetId, origin.GetProperty("object_id").GetString()); + Assert.Equal("row-1", origin.GetProperty("id").GetString()); + Assert.Equal("1042", origin.GetProperty("_xact_id").GetString()); + } + + [Fact] + public async Task InMemoryDatasetLeavesTheExperimentUnlinked() + { + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") + ); + + using var mockClient = new StubBraintrustApi(config); + + var eval = await Eval.NewBuilder() + .Name("in-memory-eval") + .Config(config) + .ApiClient(mockClient.Client) + .Cases(new DatasetCase("strawberry", "fruit")) + .TaskFunction(food => "fruit") + .Scorers(new FunctionScorer("exact", (expected, actual) => expected == actual ? 1.0 : 0.0)) + .BuildAsync(); + + await eval.RunAsync(); + + // The in-memory id is a local placeholder, not something the API would accept. + Assert.Null(mockClient.LastCreateExperimentRequest?.Dataset_id); + Assert.Null(mockClient.LastCreateExperimentRequest?.Dataset_version); + } + // ------------------------------------------------------------------------- // Test helper methods // ------------------------------------------------------------------------- + private sealed class TestDataset( + Func>> cases) : IDataset + { + public string Id => "test-dataset"; + + public string? Version => "1"; + + public IAsyncEnumerable> GetCasesAsync() => cases(); + } + /// Returns the "type" field from braintrust.span_attributes JSON. private static string? GetSpanType(Activity span) { diff --git a/tests/Braintrust.Sdk.Tests/Eval/EvalTraceTest.cs b/tests/Braintrust.Sdk.Tests/Eval/EvalTraceTest.cs index d7fbe9a..d400cfb 100644 --- a/tests/Braintrust.Sdk.Tests/Eval/EvalTraceTest.cs +++ b/tests/Braintrust.Sdk.Tests/Eval/EvalTraceTest.cs @@ -55,7 +55,7 @@ public async Task GetThreadAsync_ReturnsChronologicalMessageThread() [Fact] public async Task GetThreadAsync_ReturnsEmptyForNoLlmSpans() { - var span = MockBtqlClient.MakeSpan("task"); + var span = BtqlTestData.MakeSpan("task"); var trace = new EvalTrace(_ => Task.FromResult>>( [span])); diff --git a/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs b/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs deleted file mode 100644 index 554b09b..0000000 --- a/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs +++ /dev/null @@ -1,55 +0,0 @@ -using Braintrust.Sdk.Api; - -namespace Braintrust.Sdk.Tests.Eval; - -/// -/// Mock API client for testing that doesn't make real HTTP calls. -/// -internal class MockBraintrustApiClient : IBraintrustApiClient -{ - private readonly OrganizationInfo _orgInfo; - private readonly Project _project; - - public MockBraintrustApiClient(string orgName = "test-org", string projectName = "test-project") - { - _orgInfo = new OrganizationInfo("test-org-id", orgName); - _project = new Project("test-project-id", projectName, "test-org-id"); - } - - /// - /// The last CreateExperimentRequest received by GetOrCreateExperiment. - /// Useful for verifying that tags and metadata were passed correctly. - /// - public CreateExperimentRequest? LastCreateExperimentRequest { get; private set; } - - public Task GetOrCreateProject(string projectName) - { - return Task.FromResult(_project); - } - - public Task GetProject(string projectId) - { - return Task.FromResult(_project); - } - - public Task GetOrCreateExperiment(CreateExperimentRequest request) - { - LastCreateExperimentRequest = request; - return Task.FromResult(new Experiment("test-experiment-id", request.ProjectId, request.Name, request.Description)); - } - - public Task GetProjectAndOrgInfo() - { - return Task.FromResult(new OrganizationAndProjectInfo(_orgInfo, _project)); - } - - public Task GetProjectAndOrgInfo(string projectId) - { - return Task.FromResult(new OrganizationAndProjectInfo(_orgInfo, _project)); - } - - public Task GetOrCreateProjectAndOrgInfo() - { - return Task.FromResult(new OrganizationAndProjectInfo(_orgInfo, _project)); - } -} \ No newline at end of file diff --git a/tests/Braintrust.Sdk.Tests/Eval/MockBtqlClient.cs b/tests/Braintrust.Sdk.Tests/Eval/MockBtqlClient.cs deleted file mode 100644 index ae5539c..0000000 --- a/tests/Braintrust.Sdk.Tests/Eval/MockBtqlClient.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Text.Json; -using Braintrust.Sdk.Api.Internal; - -namespace Braintrust.Sdk.Tests.Eval; - -/// -/// Mock BTQL client for testing that returns pre-configured span data. -/// -internal class MockBtqlClient : IBtqlClient -{ - private readonly IReadOnlyList> _spans; - public int QueryCount { get; private set; } - public string? LastExperimentId { get; private set; } - public string? LastRootSpanId { get; private set; } - - public MockBtqlClient(IReadOnlyList>? spans = null) - { - _spans = spans ?? Array.Empty>(); - } - - public Task>> QuerySpansAsync( - string experimentId, string rootSpanId, CancellationToken cancellationToken = default) - { - QueryCount++; - LastExperimentId = experimentId; - LastRootSpanId = rootSpanId; - return Task.FromResult(_spans); - } - - public void Dispose() { } - - /// - /// Helper to build a span dictionary from JSON for tests. - /// - public static IReadOnlyDictionary MakeSpan(string type, object? input = null, object? output = null) - { - var obj = new Dictionary - { - ["span_id"] = Guid.NewGuid().ToString(), - ["span_attributes"] = new { type } - }; - if (input != null) obj["input"] = input; - if (output != null) obj["output"] = output; - - var json = System.Text.Json.JsonSerializer.Serialize(obj); - var doc = JsonDocument.Parse(json); - return doc.RootElement.EnumerateObject() - .ToDictionary(p => p.Name, p => p.Value.Clone()); - } -} diff --git a/tests/Braintrust.Sdk.Tests/Eval/StubBraintrustApi.cs b/tests/Braintrust.Sdk.Tests/Eval/StubBraintrustApi.cs new file mode 100644 index 0000000..3a4dd6e --- /dev/null +++ b/tests/Braintrust.Sdk.Tests/Eval/StubBraintrustApi.cs @@ -0,0 +1,218 @@ +using System.Net; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using Braintrust.Sdk.Api; +using Braintrust.Sdk.Config; +using Generated = Braintrust.Sdk.Api.Generated; + +namespace Braintrust.Sdk.Tests.Eval; + +/// +/// Stands in for the Braintrust API at the HTTP layer, so tests exercise the same generated +/// client an eval uses in production rather than a hand-written fake of it. +/// +/// Serves the three calls an eval makes at startup - resolve the project, resolve its org, create +/// the experiment - and records the experiment request that went out. +/// +internal sealed class StubBraintrustApi : IDisposable +{ + internal const string ProjectId = "3f2504e0-4f89-11d3-9a0c-0305e82c3301"; + internal const string OrgId = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; + internal const string ExperimentId = "17c1a5a0-1234-4c56-8def-0123456789ab"; + internal const string OrgName = "test-org"; + internal const string ProjectName = "test-project"; + + private readonly StubHandler _handler; + private readonly BraintrustOpenApiClient _client; + private readonly ActivityListener _activityListener; + private readonly ConcurrentDictionary _taskSpanIds = new(); + + /// + /// Org name to report. The project name is echoed back from whatever was asked for. + /// + public StubBraintrustApi( + BraintrustConfig config, + string orgName = OrgName, + IReadOnlyList>? btqlRows = null) + { + _activityListener = new ActivityListener + { + ShouldListenTo = source => source.Name == "braintrust-dotnet", + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + ActivityStopped = activity => + { + if (activity.DisplayName == "task") + { + _taskSpanIds[activity.TraceId.ToHexString()] = activity.SpanId.ToHexString(); + } + }, + }; + ActivitySource.AddActivityListener(_activityListener); + + _handler = new StubHandler(orgName, btqlRows, FindTaskSpanId); + _client = new BraintrustOpenApiClient(config, _handler, noBtqlDelay: true); + } + + /// The generated portion of the client. + public Generated.IBraintrustGeneratedApiClient Api => _client.Api; + + /// + /// The complete client, including the internal BTQL operations. + /// + public BraintrustOpenApiClient Client => _client; + + /// Paths served, so tests can assert which transport a request took. + public IReadOnlyList Paths => _handler.Paths; + + public int BtqlQueryCount => Paths.Count(path => path == "/btql"); + + /// + /// The last POST /v1/experiment body, read back through the generated model so tests + /// can assert on tags, metadata, repo info and dataset linkage as they were sent. + /// + public Generated.CreateExperiment? LastCreateExperimentRequest => _handler.LastCreateExperimentRequest; + + public void Dispose() + { + _client.Dispose(); + _handler.Dispose(); + _activityListener.Dispose(); + } + + private string? FindTaskSpanId(string query) => + _taskSpanIds.FirstOrDefault(pair => query.Contains(pair.Key, StringComparison.Ordinal)).Value; + + private sealed class StubHandler : HttpMessageHandler + { + private readonly string _orgName; + private readonly IReadOnlyList>? _btqlRows; + private readonly Func _findTaskSpanId; + + internal StubHandler( + string orgName, + IReadOnlyList>? btqlRows, + Func findTaskSpanId) + { + _orgName = orgName; + _btqlRows = btqlRows; + _findTaskSpanId = findTaskSpanId; + } + + public Generated.CreateExperiment? LastCreateExperimentRequest { get; private set; } + + public List Paths { get; } = []; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var path = request.RequestUri!.AbsolutePath; + Paths.Add(path); + var body = request.Content is null + ? null + : await request.Content.ReadAsStringAsync(cancellationToken); + + var (status, response) = Route(request.Method, path, request.RequestUri.Query, body); + return new HttpResponseMessage(status) + { + Content = new StringContent(response, Encoding.UTF8, "application/json"), + }; + } + + private (HttpStatusCode, string) Route( + HttpMethod method, + string path, + string query, + string? body) + { + if (path == "/v1/project" && method == HttpMethod.Get) + { + var name = QueryValue(query, "project_name") ?? ProjectName; + return (HttpStatusCode.OK, + $$"""{"objects":[{"id":"{{ProjectId}}","org_id":"{{OrgId}}","name":"{{name}}"}]}"""); + } + + if ((path == "/v1/project" && method == HttpMethod.Post) + || path.StartsWith("/v1/project/")) + { + var name = NameFrom(body) ?? ProjectName; + return (HttpStatusCode.OK, + $$"""{"id":"{{ProjectId}}","org_id":"{{OrgId}}","name":"{{name}}"}"""); + } + + if (path.StartsWith("/v1/organization/")) + { + return (HttpStatusCode.OK, $$"""{"id":"{{OrgId}}","name":"{{_orgName}}"}"""); + } + + if (path == "/v1/experiment" && method == HttpMethod.Post) + { + LastCreateExperimentRequest = body is null + ? null + : JsonSerializer.Deserialize(body); + + var name = NameFrom(body) ?? "test-experiment"; + return (HttpStatusCode.OK, + $$"""{"id":"{{ExperimentId}}","project_id":"{{ProjectId}}","name":"{{name}}"}"""); + } + + if (path == "/btql" && method == HttpMethod.Post) + { + var btqlQuery = JsonDocument.Parse(body!).RootElement.GetProperty("query").GetString()!; + var taskSpanId = _findTaskSpanId(btqlQuery); + var rows = (_btqlRows ?? [BtqlTestData.MakeSpan("task")]) + .Select(row => row.ToDictionary(pair => pair.Key, pair => pair.Value.Clone())) + .ToList(); + + var taskRow = rows.FirstOrDefault(row => + row.TryGetValue("span_attributes", out var attributes) + && attributes.TryGetProperty("type", out var type) + && type.GetString() == "task"); + if (taskRow is null) + { + taskRow = BtqlTestData.MakeSpan("task") + .ToDictionary(pair => pair.Key, pair => pair.Value.Clone()); + rows.Add(taskRow); + } + + if (taskSpanId is not null) + { + taskRow["span_id"] = JsonSerializer.SerializeToElement(taskSpanId); + } + + return (HttpStatusCode.OK, JsonSerializer.Serialize(new + { + data = rows, + freshness_state = new + { + last_processed_xact_id = "42", + last_considered_xact_id = "42", + }, + realtime_state = new { type = "on" }, + })); + } + + return (HttpStatusCode.NotFound, $"\"no stub for {method} {path}\""); + } + + private static string? NameFrom(string? body) + { + if (string.IsNullOrEmpty(body)) + { + return null; + } + + using var document = JsonDocument.Parse(body); + return document.RootElement.TryGetProperty("name", out var name) ? name.GetString() : null; + } + + private static string? QueryValue(string query, string key) => query + .TrimStart('?') + .Split('&', StringSplitOptions.RemoveEmptyEntries) + .Select(part => part.Split('=', 2)) + .Where(parts => parts.Length == 2 && parts[0] == key) + .Select(parts => Uri.UnescapeDataString(parts[1])) + .FirstOrDefault(); + } +} diff --git a/tests/Braintrust.Sdk.Tests/QueuedHttpHandler.cs b/tests/Braintrust.Sdk.Tests/QueuedHttpHandler.cs new file mode 100644 index 0000000..2412a4d --- /dev/null +++ b/tests/Braintrust.Sdk.Tests/QueuedHttpHandler.cs @@ -0,0 +1,40 @@ +using System.Net; +using System.Text; + +namespace Braintrust.Sdk.Tests; + +/// +/// Serves queued responses in order and records what was asked for. Call order here is +/// deterministic, so a queue is enough - the assertions check the paths. +/// +internal sealed class QueuedHttpHandler : HttpMessageHandler +{ + private readonly Queue<(HttpStatusCode Status, string Body)> _responses = new(); + + public List<(HttpMethod Method, string Path, string Query, string Body)> Requests { get; } = []; + + public void Enqueue(string body, HttpStatusCode status = HttpStatusCode.OK) + => _responses.Enqueue((status, body)); + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var body = request.Content is null + ? string.Empty + : await request.Content.ReadAsStringAsync(cancellationToken); + + Requests.Add((request.Method, request.RequestUri!.AbsolutePath, request.RequestUri.Query, body)); + + if (_responses.Count == 0) + { + throw new InvalidOperationException( + $"No response configured for {request.Method} {request.RequestUri.AbsolutePath}"); + } + + var (status, responseBody) = _responses.Dequeue(); + return new HttpResponseMessage(status) + { + Content = new StringContent(responseBody, Encoding.UTF8, "application/json"), + }; + } +}