deepseek41 : add bounded routed expert execution

common/arg.cpp

@@ -2806,6 +2806,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.ple_direct_io = value;
         }
     ).set_env("LLAMA_ARG_NGRAM_DIRECT_IO"));
+    add_opt(common_arg(
+        {"--expert-cache-slots"}, "N",
+        "DeepSeek V4.1 routed experts resident per layer; requires --expert-cache-mib",
+        [](common_params & params, int value) {
+            if (value <= 0) {
+                throw std::invalid_argument("invalid value");
+            }
+            params.expert_cache_slots = value;
+        }
+    ).set_env("LLAMA_ARG_EXPERT_CACHE_SLOTS"));
+    add_opt(common_arg(
+        {"--expert-cache-mib"}, "MiB",
+        "aggregate DeepSeek V4.1 fixed expert slot-tensor capacity; requires --expert-cache-slots",
+        [](common_params & params, int value) {
+            if (value <= 0) {
+                throw std::invalid_argument("invalid value");
+            }
+            params.expert_cache_mib = value;
+        }
+    ).set_env("LLAMA_ARG_EXPERT_CACHE_MIB"));
     add_opt(common_arg(
         {"-cmoe", "--cpu-moe"},
         "keep all Mixture of Experts (MoE) weights in the CPU",

common/common.cpp

@@ -1697,6 +1697,8 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
     mparams.ple_direct_io   = params.ple_direct_io;
     mparams.ple_io_threads  = params.ple_io_threads;
     mparams.ple_cache_mb    = params.ple_cache_mb;
+    mparams.expert_cache_slots = params.expert_cache_slots;
+    mparams.expert_cache_bytes = params.expert_cache_mib > 0 ? (size_t) params.expert_cache_mib << 20 : 0;
 
     if (params.kv_overrides.empty()) {
         mparams.kv_overrides = NULL;

common/common.h

@@ -626,6 +626,8 @@ struct common_params {
     bool    ple_direct_io  = true;  // ... read with O_DIRECT
     int32_t ple_io_threads = 64;    // ... parallel readers (random 4 KiB reads: this NVMe gives 62k IOPS at 16, 130k at 64, ~160k at 128+)
     int32_t ple_cache_mb   = 256;   // ... row cache, 0 disables
+    int32_t expert_cache_slots = 0; // DeepSeek V4.1 routed experts resident per layer
+    int32_t expert_cache_mib   = 0; // aggregate fixed slot-tensor capacity
 
     bool single_turn       = false; // single turn chat conversation
 

include/llama.h

@@ -348,6 +348,10 @@ extern "C" {
         int32_t ple_io_threads; // parallel pread workers
         int32_t ple_cache_mb;   // in-memory cache of recently read rows, 0 disables
 
+        // DeepSeek V4.1 routed-expert cache. Both values must be non-zero.
+        size_t  expert_cache_bytes;
+        int32_t expert_cache_slots;
+
         // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
         const float * tensor_split;
 

src/CMakeLists.txt

@@ -18,6 +18,8 @@ set(LLAMA_CORE_SOURCES
     llama-cparams.cpp
     llama-dsv41.cpp
     llama-dsv41-engram.cpp
+    llama-dsv41-expert.cpp
+    llama-expert-store.cpp
     llama-grammar.cpp
     llama-graph.cpp
     llama-hparams.cpp

src/llama-context.cpp

@@ -481,11 +481,15 @@ llama_context::llama_context(
             sampling.token_ids_full_vocab[i] = i;
         }
     }
+
+    model.acquire_runtime_context();
 }
 
 llama_context::~llama_context() {
     // wait for any pending asynchronous copies into the output buffers before they are freed
     synchronize();
+    model.release_runtime_work();
+    model.release_runtime_context();
 
     // when training, ggml_opt allocates extra buffers through the scheduler, so the sizes no longer match the expectation
     if (!model.hparams.no_alloc && !opt_ctx) {
@@ -1422,10 +1426,21 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll
 
     const auto status = graph_compute(res->get_gf(), ubatch.n_tokens > 1);
     if (status != GGML_STATUS_SUCCESS) {
+        model.release_runtime_work_after_sync(sched.get());
         LLAMA_LOG_ERROR("%s: failed to compute graph, compute status: %d\n", __func__, status);
         ret = status;
         return nullptr;
     }
+    if (model.requires_synchronous_graph()) {
+        synchronize();
+        const std::string error = model.consume_runtime_error();
+        if (!error.empty()) {
+            model.release_runtime_work();
+            LLAMA_LOG_ERROR("%s: model runtime failed: %s\n", __func__, error.c_str());
+            ret = GGML_STATUS_FAILED;
+            return nullptr;
+        }
+    }
 
     ret = GGML_STATUS_SUCCESS;