Enable chunked prefill by default for supported execution providers

sdk_v2/cpp/src/inferencing/execution_provider.h

@@ -23,9 +23,12 @@ enum class ExecutionProvider {
 struct EPUtils {
   /// Convert a value from the catalog, genai config, or EP override param to an ExecutionProvider
   /// See NormalizeProviderName https://github.com/microsoft/onnxruntime-genai/blob/main/src/config.cpp
-  /// See AppendExecutionProviderV1 https://github.com/microsoft/onnxruntime-genai/blob/main/src/models/session_options.cpp
+  /// See AppendExecutionProviderV1
+  /// https://github.com/microsoft/onnxruntime-genai/blob/main/src/models/session_options.cpp
   static ExecutionProvider StringtoEP(std::string_view ep) {
-    if (ep == "CPUExecutionProvider") {
+    if (ep == "cpu" ||
+        ep == "CPU" ||
+        ep == "CPUExecutionProvider") {
       return ExecutionProvider::kCPU;
     } else if (ep == "cuda" ||
                ep == "CUDA" ||
@@ -53,8 +56,6 @@ struct EPUtils {
                ep == "QNN" ||
                ep == "QNNExecutionProvider") {
       return ExecutionProvider::kQNN;
-      // } else if (genai_ep == "cpu") {
-      //   if CPU there's no provider set in the config and this shouldn't be called
     } else {
       return ExecutionProvider::kUnknown;
     }

sdk_v2/cpp/src/inferencing/generative/chat/onnx_chat_generator.cc

@@ -304,8 +304,8 @@ std::unique_ptr<OnnxChatGenerator> OnnxChatGenerator::CreateImpl(const std::vect
   //    Default output budget mirrors C# OnnxChatGenerator: 3072 for vision requests
   //    (image tokens push the prompt much higher), 2048 for text.
   int default_max_output = vision_branch ? 3072 : 2048;
-  ApplySearchOptions(options, input_token_count, model.GetGenAIConfig(), *gen_params, use_full_context,
-                     default_max_output);
+  ApplySearchOptions(options, input_token_count, model.GetGenAIConfig(), *gen_params, model.EP(),
+                     use_full_context, default_max_output);
 
   // 5. Compute guidance for constrained decoding.
   // Priority: user-specified guidance (from response_format) > auto-generated LARK grammar.

sdk_v2/cpp/src/inferencing/generative/chat/search_options.cc

@@ -14,6 +14,7 @@ int ApplySearchOptions(const SearchOptions& options,
                        int input_token_count,
                        const GenAIConfig& config,
                        OgaGeneratorParams& gen_params,
+                       ExecutionProvider ep,
                        bool use_full_context,
                        int default_max_output_tokens) {
   // Determine model's max context length from genai_config.json search.max_length
@@ -95,6 +96,31 @@ int ApplySearchOptions(const SearchOptions& options,
     gen_params.SetSearchOptionBool("early_stopping", true);
   }
 
+  // Preserve a positive model setting. ORT GenAI reports both an absent setting and explicit zero as zero; Foundry
+  // Local intentionally treats both as unset. ORT GenAI decides whether the model consumes the resulting option.
+  if (gen_params.GetSearchNumber("chunk_size") <= 0) {
+    // The model's resolved EP is kDefault for the common load path, so use the provider declared in
+    // genai_config.json. An empty provider means ORT's CPU fallback.
+    ExecutionProvider effective_ep = ep;
+    if (effective_ep == ExecutionProvider::kDefault) {
+      std::string config_provider = config.DefaultProvider();
+      effective_ep = config_provider.empty() ? ExecutionProvider::kCPU
+                                              : EPUtils::StringtoEP(config_provider);
+    }
+
+    constexpr double kDefaultChunkSize = 2048.0;
+    switch (effective_ep) {
+      case ExecutionProvider::kCUDA:
+      case ExecutionProvider::kTensorRT_RTX:
+      case ExecutionProvider::kWebGPU:
+      case ExecutionProvider::kCPU:
+        gen_params.SetSearchOption("chunk_size", kDefaultChunkSize);
+        break;
+      default:
+        break;
+    }
+  }
+
   return effective_max_length;
 }
 

sdk_v2/cpp/src/inferencing/generative/chat/search_options.h

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 #pragma once
 
+#include "inferencing/execution_provider.h"
 #include "inferencing/generative/genai_config.h"
 #include "util/key_value_pairs.h"
 
@@ -57,6 +58,11 @@ struct SearchOptions {
 /// @param input_token_count  Number of tokens in the encoded prompt
 /// @param config             Model's GenAI config (for search.max_length)
 /// @param gen_params         ORT GenAI generator params to configure
+/// @param ep                 Resolved execution provider. Used to enable chunked prefill by default
+///                           on providers that benefit from it (CUDA, NvTensorRtRtx, WebGPU, CPU).
+///                           When kDefault, the effective provider is taken from
+///                           config.DefaultProvider() (empty ⇒ CPU).
+///                           ORT GenAI determines whether the model consumes this option.
 /// @param use_full_context   When true, set max_length to the model's full context window
 ///                           instead of input+output. Used for continuous decoding (cached generators).
 /// @param default_max_output_tokens  Default applied when the request does not specify
@@ -65,6 +71,7 @@ int ApplySearchOptions(const SearchOptions& options,
                        int input_token_count,
                        const GenAIConfig& config,
                        OgaGeneratorParams& gen_params,
+                       ExecutionProvider ep,
                        bool use_full_context = false,
                        int default_max_output_tokens = 2048);
 

sdk_v2/cpp/test/internal_api/chat/search_options_test.cc

@@ -75,7 +75,7 @@ TEST_F(SearchOptionsTest, DefaultOptionsApplySuccessfully) {
   SearchOptions opts;
   auto params = MakeParams();
 
-  int max_length = ApplySearchOptions(opts, 10, GetConfig(), *params);
+  int max_length = ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault);
   EXPECT_GT(max_length, 10);
   // Default output tokens = 2048, so max_length should be 10 + 2048 = 2058
   EXPECT_EQ(max_length, 2058);
@@ -86,7 +86,7 @@ TEST_F(SearchOptionsTest, MaxOutputTokensRespected) {
   opts.max_output_tokens = 100;
   auto params = MakeParams();
 
-  int max_length = ApplySearchOptions(opts, 50, GetConfig(), *params);
+  int max_length = ApplySearchOptions(opts, 50, GetConfig(), *params, ExecutionProvider::kDefault);
   EXPECT_EQ(max_length, 150);  // 50 input + 100 output
 }
 
@@ -96,7 +96,8 @@ TEST_F(SearchOptionsTest, TokenBudgetExceededThrows) {
   opts.max_output_tokens = 32000;
   auto params = MakeParams();
 
-  EXPECT_THROW(ApplySearchOptions(opts, 1000, GetConfig(), *params), fl::Exception);
+  EXPECT_THROW(ApplySearchOptions(opts, 1000, GetConfig(), *params, ExecutionProvider::kDefault),
+               fl::Exception);
 }
 
 TEST_F(SearchOptionsTest, TemperatureZeroDisablesSampling) {
@@ -105,15 +106,15 @@ TEST_F(SearchOptionsTest, TemperatureZeroDisablesSampling) {
   auto params = MakeParams();
 
   // Should not throw — temperature 0 → do_sample=false
-  EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params));
+  EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault));
 }
 
 TEST_F(SearchOptionsTest, TemperaturePositiveEnablesSampling) {
   SearchOptions opts;
   opts.temperature = 0.7f;
   auto params = MakeParams();
 
-  EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params));
+  EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault));
 }
 
 TEST_F(SearchOptionsTest, AllOptionsSetSimultaneously) {
@@ -128,7 +129,7 @@ TEST_F(SearchOptionsTest, AllOptionsSetSimultaneously) {
   opts.do_sample = true;
   auto params = MakeParams();
 
-  int max_length = ApplySearchOptions(opts, 20, GetConfig(), *params);
+  int max_length = ApplySearchOptions(opts, 20, GetConfig(), *params, ExecutionProvider::kDefault);
   EXPECT_EQ(max_length, 276);  // 20 + 256
 }
 
@@ -137,15 +138,17 @@ TEST_F(SearchOptionsTest, ZeroMaxOutputTokensThrows) {
   opts.max_output_tokens = 0;
   auto params = MakeParams();
 
-  EXPECT_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params), fl::Exception);
+  EXPECT_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault),
+               fl::Exception);
 }
 
 TEST_F(SearchOptionsTest, NegativeMaxOutputTokensThrows) {
   SearchOptions opts;
   opts.max_output_tokens = -5;
   auto params = MakeParams();
 
-  EXPECT_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params), fl::Exception);
+  EXPECT_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault),
+               fl::Exception);
 }
 
 TEST_F(SearchOptionsTest, ExplicitDoSampleOverridesTemperature) {
@@ -154,7 +157,7 @@ TEST_F(SearchOptionsTest, ExplicitDoSampleOverridesTemperature) {
   opts.do_sample = true;    // But explicit override takes priority
   auto params = MakeParams();
 
-  EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params));
+  EXPECT_NO_THROW(ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kDefault));
 }
 
 TEST_F(SearchOptionsTest, LargeInputFitsExactly) {
@@ -163,7 +166,7 @@ TEST_F(SearchOptionsTest, LargeInputFitsExactly) {
   opts.max_output_tokens = 768;
   auto params = MakeParams();
 
-  int max_length = ApplySearchOptions(opts, 32000, GetConfig(), *params);
+  int max_length = ApplySearchOptions(opts, 32000, GetConfig(), *params, ExecutionProvider::kDefault);
   EXPECT_EQ(max_length, 32768);  // Exactly at limit
 }
 
@@ -172,5 +175,59 @@ TEST_F(SearchOptionsTest, LargeInputExceedsByOneThrows) {
   opts.max_output_tokens = 769;
   auto params = MakeParams();
 
-  EXPECT_THROW(ApplySearchOptions(opts, 32000, GetConfig(), *params), fl::Exception);
+  EXPECT_THROW(ApplySearchOptions(opts, 32000, GetConfig(), *params, ExecutionProvider::kDefault),
+               fl::Exception);
+}
+
+TEST_F(SearchOptionsTest, ChunkedPrefillDefaultsTo2048ForSupportedExecutionProviders) {
+  SearchOptions opts;
+
+  for (ExecutionProvider ep : {ExecutionProvider::kCPU, ExecutionProvider::kCUDA,
+                               ExecutionProvider::kTensorRT_RTX, ExecutionProvider::kWebGPU}) {
+    auto params = MakeParams();
+    ApplySearchOptions(opts, 10, GetConfig(), *params, ep);
+    EXPECT_EQ(params->GetSearchNumber("chunk_size"), 2048) << "EP: " << static_cast<int>(ep);
+  }
+}
+
+TEST_F(SearchOptionsTest, ChunkedPrefillSkippedForUnlistedEp) {
+  SearchOptions opts;
+  auto params = MakeParams();
+
+  ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kOpenVINO);
+  EXPECT_EQ(params->GetSearchNumber("chunk_size"), 0);
+}
+
+TEST_F(SearchOptionsTest, ChunkedPrefillResolvesSupportedProvidersFromConfig) {
+  SearchOptions opts;
+  GenAIConfig config;
+  auto& model = config.model.emplace();
+  auto& decoder = model.decoder.emplace();
+  auto& session_options = decoder.session_options.emplace();
+  config.search.emplace().max_length = 32768;
+
+  for (const char* provider : {"cpu", "cuda", "NvTensorRtRtx", "WebGPU"}) {
+    session_options.provider_options = {{{provider, "{}"}}};
+    auto params = MakeParams();
+    ApplySearchOptions(opts, 10, config, *params, ExecutionProvider::kDefault);
+    EXPECT_EQ(params->GetSearchNumber("chunk_size"), 2048) << "Provider: " << provider;
+  }
+}
+
+TEST_F(SearchOptionsTest, ChunkedPrefillPreservesModelSetting) {
+  SearchOptions opts;
+  auto params = MakeParams();
+  params->SetSearchOption("chunk_size", 1024);
+
+  ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kCPU);
+  EXPECT_EQ(params->GetSearchNumber("chunk_size"), 1024);
+}
+
+TEST_F(SearchOptionsTest, ChunkedPrefillTreatsZeroAsUnset) {
+  SearchOptions opts;
+  auto params = MakeParams();
+  params->SetSearchOption("chunk_size", 0);
+
+  ApplySearchOptions(opts, 10, GetConfig(), *params, ExecutionProvider::kCPU);
+  EXPECT_EQ(params->GetSearchNumber("chunk_size"), 2048);
 }

sdk_v2/cpp/test/internal_api/execution_provider_test.cc

@@ -7,6 +7,8 @@
 using namespace fl;
 
 TEST(ExecutionProviderTest, StringToEPRecognizesSupportedNames) {
+  EXPECT_EQ(EPUtils::StringtoEP("cpu"), ExecutionProvider::kCPU);
+  EXPECT_EQ(EPUtils::StringtoEP("CPU"), ExecutionProvider::kCPU);
   EXPECT_EQ(EPUtils::StringtoEP("CPUExecutionProvider"), ExecutionProvider::kCPU);
   EXPECT_EQ(EPUtils::StringtoEP("cuda"), ExecutionProvider::kCUDA);
   EXPECT_EQ(EPUtils::StringtoEP("CUDAExecutionProvider"), ExecutionProvider::kCUDA);
@@ -20,7 +22,6 @@ TEST(ExecutionProviderTest, StringToEPRecognizesSupportedNames) {
 
 TEST(ExecutionProviderTest, StringToEPReturnsUnknownForUnsupportedOrEmptyNames) {
   EXPECT_EQ(EPUtils::StringtoEP(""), ExecutionProvider::kUnknown);
-  EXPECT_EQ(EPUtils::StringtoEP("cpu"), ExecutionProvider::kUnknown);
   EXPECT_EQ(EPUtils::StringtoEP("trt"), ExecutionProvider::kUnknown);
   EXPECT_EQ(EPUtils::StringtoEP("DirectMLExecutionProvider"), ExecutionProvider::kUnknown);
 }

sdk_v2/cpp/test/sdk_api/chat_session_test.cc

@@ -130,54 +130,18 @@ TEST_F(ModelFixture, SessionSetOptionsAcceptsRequestOptions) {
       << "Expected non-empty output with session-level RequestOptions applied.";
 }
 
-// Multi-turn E2E test: exercises generator caching and delayed history commit
-// across multiple ProcessRequest calls on the same session.
-//
-// NOTE ON SOFT ARITHMETIC CHECKS
-// ------------------------------
-// This test's job is to validate SESSION MECHANICS (cached-generator reuse,
-// delayed history commit, and undo/replay), not the model's arithmetic quality.
-// The exact answer digit a small (0.5B) greedy model emits is not stable across
-// CPU microarchitectures: onnxruntime 1.28 routes FP32 GroupQueryAttention
-// single-token decode through a new flash / online-softmax GEMV kernel whose
-// softmax reductions (MlasReduceMaximumF32Kernel / MlasComputeSumExpF32Kernel)
-// dispatch to AVX-512F variants on AVX-512 hosts (e.g. AMD EPYC 9V74) and to
-// AVX2 variants elsewhere. Those bind different floating-point accumulation
-// orders, so a few-ULP delta can flip the greedy argmax between near-tie digit
-// tokens (observed in CI: turn 2 '5'->'3', turn 4 '6'->'2'). This is
-// mathematically-equivalent FP reordering, not an SDK bug, so we do NOT hard-fail
-// on the exact digit here. The mechanics are still asserted strictly below: the
-// finish reason, token counts, turn count, and — most importantly — the
-// turn-3-equals-turn-2 rewind-determinism check (same host, same code path, so
-// bit-stable). The exact-digit expectations are emitted as non-fatal warnings.
-//
-// Other (non-invasive) options considered, in rough order of preference:
-//   1. Assert numerical correctness at the ORT/kernel level instead of here (best
-//      fit): pin the decode path in an ORT-level determinism test and keep SDK
-//      tests focused on mechanics. Preferred long-term home for this check.
-//   2. Widen the logit margin so greedy is not a near-tie — e.g. prompt for a
-//      spelled-out or multi-token answer, or a question whose correct token
-//      dominates — keeping a hard content assertion that is ISA-robust.
-//   3. Force the legacy attention path for deterministic runs by setting
-//      ORT_GQA_DISABLE_FLASH_ATTENTION=1 in the test/CI environment (keeps the
-//      fast flash path in production; only affects the test process).
-//   4. Accept a small set of plausible answers (tolerance) rather than one exact
-//      digit.
-// We take the least-invasive route (1-line soft checks) here; options 1-3 remain
-// open if stricter arithmetic validation is wanted.
+// Exercises cached-generator reuse, delayed history commit, and undo/replay. Exact arithmetic outputs are
+// soft-checked because greedy decoding can vary across CPU architectures; session mechanics remain strict.
 TEST_F(ModelFixture, ChatMultiTurnSession) {
   using namespace foundry_local;
 
-  // Soft, non-fatal content check: logs a warning but does not fail the test when
-  // the model emits an unexpected digit due to the cross-ISA FP nondeterminism
-  // described above. Mechanics assertions below remain strict (EXPECT_*).
+  // Keep content checks non-fatal while asserting session mechanics strictly below.
   auto expect_contains_soft = [](const std::string& haystack, const std::string& needle,
                                  const std::string& context) {
     if (haystack.find(needle) == std::string::npos) {
       GTEST_LOG_(WARNING) << context << ": expected '" << needle
                           << "' but model emitted '" << haystack
-                          << "'. Treated as non-fatal (cross-ISA greedy-decode FP nondeterminism; "
-                          << "see note above ChatMultiTurnSession).";
+                          << "'. Treated as non-fatal due to cross-ISA greedy-decode differences.";
     }
   };