diff --git a/sdk_v2/cpp/src/inferencing/execution_provider.h b/sdk_v2/cpp/src/inferencing/execution_provider.h index 2dfd615b..b1e06295 100644 --- a/sdk_v2/cpp/src/inferencing/execution_provider.h +++ b/sdk_v2/cpp/src/inferencing/execution_provider.h @@ -23,9 +23,12 @@ enum class ExecutionProvider { struct EPUtils { /// Convert a value from the catalog, genai config, or EP override param to an ExecutionProvider /// See NormalizeProviderName https://github.com/microsoft/onnxruntime-genai/blob/main/src/config.cpp - /// See AppendExecutionProviderV1 https://github.com/microsoft/onnxruntime-genai/blob/main/src/models/session_options.cpp + /// See AppendExecutionProviderV1 + /// https://github.com/microsoft/onnxruntime-genai/blob/main/src/models/session_options.cpp static ExecutionProvider StringtoEP(std::string_view ep) { - if (ep == "CPUExecutionProvider") { + if (ep == "cpu" || + ep == "CPU" || + ep == "CPUExecutionProvider") { return ExecutionProvider::kCPU; } else if (ep == "cuda" || ep == "CUDA" || @@ -53,8 +56,6 @@ struct EPUtils { ep == "QNN" || ep == "QNNExecutionProvider") { return ExecutionProvider::kQNN; - // } else if (genai_ep == "cpu") { - // if CPU there's no provider set in the config and this shouldn't be called } else { return ExecutionProvider::kUnknown; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc index 3a6875f1..355b9005 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc @@ -304,8 +304,8 @@ std::unique_ptr OnnxChatGenerator::CreateImpl(const std::vect // Default output budget mirrors C# OnnxChatGenerator: 3072 for vision requests // (image tokens push the prompt much higher), 2048 for text. int default_max_output = vision_branch ? 3072 : 2048; - ApplySearchOptions(options, input_token_count, model.GetGenAIConfig(), *gen_params, use_full_context, - default_max_output); + ApplySearchOptions(options, input_token_count, model.GetGenAIConfig(), *gen_params, model.EP(), + use_full_context, default_max_output); // 5. Compute guidance for constrained decoding. // Priority: user-specified guidance (from response_format) > auto-generated LARK grammar. diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc index 22b060b4..5ac83bdf 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc +++ b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc @@ -14,6 +14,7 @@ int ApplySearchOptions(const SearchOptions& options, int input_token_count, const GenAIConfig& config, OgaGeneratorParams& gen_params, + ExecutionProvider ep, bool use_full_context, int default_max_output_tokens) { // Determine model's max context length from genai_config.json search.max_length @@ -95,6 +96,31 @@ int ApplySearchOptions(const SearchOptions& options, gen_params.SetSearchOptionBool("early_stopping", true); } + // Preserve a positive model setting. ORT GenAI reports both an absent setting and explicit zero as zero; Foundry + // Local intentionally treats both as unset. ORT GenAI decides whether the model consumes the resulting option. + if (gen_params.GetSearchNumber("chunk_size") <= 0) { + // The model's resolved EP is kDefault for the common load path, so use the provider declared in + // genai_config.json. An empty provider means ORT's CPU fallback. + ExecutionProvider effective_ep = ep; + if (effective_ep == ExecutionProvider::kDefault) { + std::string config_provider = config.DefaultProvider(); + effective_ep = config_provider.empty() ? ExecutionProvider::kCPU + : EPUtils::StringtoEP(config_provider); + } + + constexpr double kDefaultChunkSize = 2048.0; + switch (effective_ep) { + case ExecutionProvider::kCUDA: + case ExecutionProvider::kTensorRT_RTX: + case ExecutionProvider::kWebGPU: + case ExecutionProvider::kCPU: + gen_params.SetSearchOption("chunk_size", kDefaultChunkSize); + break; + default: + break; + } + } + return effective_max_length; } diff --git a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.h b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.h index 33eef7fe..7bc162c8 100644 --- a/sdk_v2/cpp/src/inferencing/generative/chat/search_options.h +++ b/sdk_v2/cpp/src/inferencing/generative/chat/search_options.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once +#include "inferencing/execution_provider.h" #include "inferencing/generative/genai_config.h" #include "util/key_value_pairs.h" @@ -57,6 +58,11 @@ struct SearchOptions { /// @param input_token_count Number of tokens in the encoded prompt /// @param config Model's GenAI config (for search.max_length) /// @param gen_params ORT GenAI generator params to configure +/// @param ep Resolved execution provider. Used to enable chunked prefill by default +/// on providers that benefit from it (CUDA, NvTensorRtRtx, WebGPU, CPU). +/// When kDefault, the effective provider is taken from +/// config.DefaultProvider() (empty ⇒ CPU). +/// ORT GenAI determines whether the model consumes this option. /// @param use_full_context When true, set max_length to the model's full context window /// instead of input+output. Used for continuous decoding (cached generators). /// @param default_max_output_tokens Default applied when the request does not specify @@ -65,6 +71,7 @@ int ApplySearchOptions(const SearchOptions& options, int input_token_count, const GenAIConfig& config, OgaGeneratorParams& gen_params, + ExecutionProvider ep, bool use_full_context = false, int default_max_output_tokens = 2048); diff --git a/sdk_v2/cpp/test/internal_api/chat/search_options_test.cc b/sdk_v2/cpp/test/internal_api/chat/search_options_test.cc index 0c9ca3cf..a9c9172e 100644 --- a/sdk_v2/cpp/test/internal_api/chat/search_options_test.cc +++ b/sdk_v2/cpp/test/internal_api/chat/search_options_test.cc @@ -75,7 +75,7 @@ TEST_F(SearchOptionsTest, DefaultOptionsApplySuccessfully) { SearchOptions opts; auto params = MakeParams(); - int max_length = ApplySearchOptions(opts, 10, GetConfig(), *params); + int max_length = ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault); EXPECT_GT(max_length, 10); // Default output tokens = 2048, so max_length should be 10 + 2048 = 2058 EXPECT_EQ(max_length, 2058); @@ -86,7 +86,7 @@ TEST_F(SearchOptionsTest, MaxOutputTokensRespected) { opts.max_output_tokens = 100; auto params = MakeParams(); - int max_length = ApplySearchOptions(opts, 50, GetConfig(), *params); + int max_length = ApplySearchOptions(opts, 50, GetConfig(), *params, ExecutionProvider::kDefault); EXPECT_EQ(max_length, 150); // 50 input + 100 output } @@ -96,7 +96,8 @@ TEST_F(SearchOptionsTest, TokenBudgetExceededThrows) { opts.max_output_tokens = 32000; auto params = MakeParams(); - EXPECT_THROW(ApplySearchOptions(opts, 1000, GetConfig(), *params), fl::Exception); + EXPECT_THROW(ApplySearchOptions(opts, 1000, GetConfig(), *params, ExecutionProvider::kDefault), + fl::Exception); } TEST_F(SearchOptionsTest, TemperatureZeroDisablesSampling) { @@ -105,7 +106,7 @@ TEST_F(SearchOptionsTest, TemperatureZeroDisablesSampling) { auto params = MakeParams(); // Should not throw — temperature 0 → do_sample=false - EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params)); + EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault)); } TEST_F(SearchOptionsTest, TemperaturePositiveEnablesSampling) { @@ -113,7 +114,7 @@ TEST_F(SearchOptionsTest, TemperaturePositiveEnablesSampling) { opts.temperature = 0.7f; auto params = MakeParams(); - EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params)); + EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault)); } TEST_F(SearchOptionsTest, AllOptionsSetSimultaneously) { @@ -128,7 +129,7 @@ TEST_F(SearchOptionsTest, AllOptionsSetSimultaneously) { opts.do_sample = true; auto params = MakeParams(); - int max_length = ApplySearchOptions(opts, 20, GetConfig(), *params); + int max_length = ApplySearchOptions(opts, 20, GetConfig(), *params, ExecutionProvider::kDefault); EXPECT_EQ(max_length, 276); // 20 + 256 } @@ -137,7 +138,8 @@ TEST_F(SearchOptionsTest, ZeroMaxOutputTokensThrows) { opts.max_output_tokens = 0; auto params = MakeParams(); - EXPECT_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params), fl::Exception); + EXPECT_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault), + fl::Exception); } TEST_F(SearchOptionsTest, NegativeMaxOutputTokensThrows) { @@ -145,7 +147,8 @@ TEST_F(SearchOptionsTest, NegativeMaxOutputTokensThrows) { opts.max_output_tokens = -5; auto params = MakeParams(); - EXPECT_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params), fl::Exception); + EXPECT_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault), + fl::Exception); } TEST_F(SearchOptionsTest, ExplicitDoSampleOverridesTemperature) { @@ -154,7 +157,7 @@ TEST_F(SearchOptionsTest, ExplicitDoSampleOverridesTemperature) { opts.do_sample = true; // But explicit override takes priority auto params = MakeParams(); - EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params)); + EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault)); } TEST_F(SearchOptionsTest, LargeInputFitsExactly) { @@ -163,7 +166,7 @@ TEST_F(SearchOptionsTest, LargeInputFitsExactly) { opts.max_output_tokens = 768; auto params = MakeParams(); - int max_length = ApplySearchOptions(opts, 32000, GetConfig(), *params); + int max_length = ApplySearchOptions(opts, 32000, GetConfig(), *params, ExecutionProvider::kDefault); EXPECT_EQ(max_length, 32768); // Exactly at limit } @@ -172,5 +175,59 @@ TEST_F(SearchOptionsTest, LargeInputExceedsByOneThrows) { opts.max_output_tokens = 769; auto params = MakeParams(); - EXPECT_THROW(ApplySearchOptions(opts, 32000, GetConfig(), *params), fl::Exception); + EXPECT_THROW(ApplySearchOptions(opts, 32000, GetConfig(), *params, ExecutionProvider::kDefault), + fl::Exception); +} + +TEST_F(SearchOptionsTest, ChunkedPrefillDefaultsTo2048ForSupportedExecutionProviders) { + SearchOptions opts; + + for (ExecutionProvider ep : {ExecutionProvider::kCPU, ExecutionProvider::kCUDA, + ExecutionProvider::kTensorRT_RTX, ExecutionProvider::kWebGPU}) { + auto params = MakeParams(); + ApplySearchOptions(opts, 10, GetConfig(), *params, ep); + EXPECT_EQ(params->GetSearchNumber("chunk_size"), 2048) << "EP: " << static_cast(ep); + } +} + +TEST_F(SearchOptionsTest, ChunkedPrefillSkippedForUnlistedEp) { + SearchOptions opts; + auto params = MakeParams(); + + ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kOpenVINO); + EXPECT_EQ(params->GetSearchNumber("chunk_size"), 0); +} + +TEST_F(SearchOptionsTest, ChunkedPrefillResolvesSupportedProvidersFromConfig) { + SearchOptions opts; + GenAIConfig config; + auto& model = config.model.emplace(); + auto& decoder = model.decoder.emplace(); + auto& session_options = decoder.session_options.emplace(); + config.search.emplace().max_length = 32768; + + for (const char* provider : {"cpu", "cuda", "NvTensorRtRtx", "WebGPU"}) { + session_options.provider_options = {{{provider, "{}"}}}; + auto params = MakeParams(); + ApplySearchOptions(opts, 10, config, *params, ExecutionProvider::kDefault); + EXPECT_EQ(params->GetSearchNumber("chunk_size"), 2048) << "Provider: " << provider; + } +} + +TEST_F(SearchOptionsTest, ChunkedPrefillPreservesModelSetting) { + SearchOptions opts; + auto params = MakeParams(); + params->SetSearchOption("chunk_size", 1024); + + ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kCPU); + EXPECT_EQ(params->GetSearchNumber("chunk_size"), 1024); +} + +TEST_F(SearchOptionsTest, ChunkedPrefillTreatsZeroAsUnset) { + SearchOptions opts; + auto params = MakeParams(); + params->SetSearchOption("chunk_size", 0); + + ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kCPU); + EXPECT_EQ(params->GetSearchNumber("chunk_size"), 2048); } diff --git a/sdk_v2/cpp/test/internal_api/execution_provider_test.cc b/sdk_v2/cpp/test/internal_api/execution_provider_test.cc index 4df9aa42..c31f0139 100644 --- a/sdk_v2/cpp/test/internal_api/execution_provider_test.cc +++ b/sdk_v2/cpp/test/internal_api/execution_provider_test.cc @@ -7,6 +7,8 @@ using namespace fl; TEST(ExecutionProviderTest, StringToEPRecognizesSupportedNames) { + EXPECT_EQ(EPUtils::StringtoEP("cpu"), ExecutionProvider::kCPU); + EXPECT_EQ(EPUtils::StringtoEP("CPU"), ExecutionProvider::kCPU); EXPECT_EQ(EPUtils::StringtoEP("CPUExecutionProvider"), ExecutionProvider::kCPU); EXPECT_EQ(EPUtils::StringtoEP("cuda"), ExecutionProvider::kCUDA); EXPECT_EQ(EPUtils::StringtoEP("CUDAExecutionProvider"), ExecutionProvider::kCUDA); @@ -20,7 +22,6 @@ TEST(ExecutionProviderTest, StringToEPRecognizesSupportedNames) { TEST(ExecutionProviderTest, StringToEPReturnsUnknownForUnsupportedOrEmptyNames) { EXPECT_EQ(EPUtils::StringtoEP(""), ExecutionProvider::kUnknown); - EXPECT_EQ(EPUtils::StringtoEP("cpu"), ExecutionProvider::kUnknown); EXPECT_EQ(EPUtils::StringtoEP("trt"), ExecutionProvider::kUnknown); EXPECT_EQ(EPUtils::StringtoEP("DirectMLExecutionProvider"), ExecutionProvider::kUnknown); } diff --git a/sdk_v2/cpp/test/sdk_api/chat_session_test.cc b/sdk_v2/cpp/test/sdk_api/chat_session_test.cc index c651f428..9a9b6589 100644 --- a/sdk_v2/cpp/test/sdk_api/chat_session_test.cc +++ b/sdk_v2/cpp/test/sdk_api/chat_session_test.cc @@ -130,54 +130,18 @@ TEST_F(ModelFixture, SessionSetOptionsAcceptsRequestOptions) { << "Expected non-empty output with session-level RequestOptions applied."; } -// Multi-turn E2E test: exercises generator caching and delayed history commit -// across multiple ProcessRequest calls on the same session. -// -// NOTE ON SOFT ARITHMETIC CHECKS -// ------------------------------ -// This test's job is to validate SESSION MECHANICS (cached-generator reuse, -// delayed history commit, and undo/replay), not the model's arithmetic quality. -// The exact answer digit a small (0.5B) greedy model emits is not stable across -// CPU microarchitectures: onnxruntime 1.28 routes FP32 GroupQueryAttention -// single-token decode through a new flash / online-softmax GEMV kernel whose -// softmax reductions (MlasReduceMaximumF32Kernel / MlasComputeSumExpF32Kernel) -// dispatch to AVX-512F variants on AVX-512 hosts (e.g. AMD EPYC 9V74) and to -// AVX2 variants elsewhere. Those bind different floating-point accumulation -// orders, so a few-ULP delta can flip the greedy argmax between near-tie digit -// tokens (observed in CI: turn 2 '5'->'3', turn 4 '6'->'2'). This is -// mathematically-equivalent FP reordering, not an SDK bug, so we do NOT hard-fail -// on the exact digit here. The mechanics are still asserted strictly below: the -// finish reason, token counts, turn count, and — most importantly — the -// turn-3-equals-turn-2 rewind-determinism check (same host, same code path, so -// bit-stable). The exact-digit expectations are emitted as non-fatal warnings. -// -// Other (non-invasive) options considered, in rough order of preference: -// 1. Assert numerical correctness at the ORT/kernel level instead of here (best -// fit): pin the decode path in an ORT-level determinism test and keep SDK -// tests focused on mechanics. Preferred long-term home for this check. -// 2. Widen the logit margin so greedy is not a near-tie — e.g. prompt for a -// spelled-out or multi-token answer, or a question whose correct token -// dominates — keeping a hard content assertion that is ISA-robust. -// 3. Force the legacy attention path for deterministic runs by setting -// ORT_GQA_DISABLE_FLASH_ATTENTION=1 in the test/CI environment (keeps the -// fast flash path in production; only affects the test process). -// 4. Accept a small set of plausible answers (tolerance) rather than one exact -// digit. -// We take the least-invasive route (1-line soft checks) here; options 1-3 remain -// open if stricter arithmetic validation is wanted. +// Exercises cached-generator reuse, delayed history commit, and undo/replay. Exact arithmetic outputs are +// soft-checked because greedy decoding can vary across CPU architectures; session mechanics remain strict. TEST_F(ModelFixture, ChatMultiTurnSession) { using namespace foundry_local; - // Soft, non-fatal content check: logs a warning but does not fail the test when - // the model emits an unexpected digit due to the cross-ISA FP nondeterminism - // described above. Mechanics assertions below remain strict (EXPECT_*). + // Keep content checks non-fatal while asserting session mechanics strictly below. auto expect_contains_soft = [](const std::string& haystack, const std::string& needle, const std::string& context) { if (haystack.find(needle) == std::string::npos) { GTEST_LOG_(WARNING) << context << ": expected '" << needle << "' but model emitted '" << haystack - << "'. Treated as non-fatal (cross-ISA greedy-decode FP nondeterminism; " - << "see note above ChatMultiTurnSession)."; + << "'. Treated as non-fatal due to cross-ISA greedy-decode differences."; } };