From f2e1a8f3b2c904d26bf2dec47baa82b9c9fec295 Mon Sep 17 00:00:00 2001 From: piggidragon Date: Tue, 1 Sep 2026 21:41:48 +0200 Subject: [PATCH 1/4] server : allow cache reuse for text-only prompts with mmproj loaded has_mtmd only means an mmproj is loaded, not that the current prompt carries media. Loading an mmproj disabled cache reuse for the whole server, so text-only requests lost prompt cache reuse too. Gate cache reuse on real media chunks in the cached prompt and the new prompt instead. It stays disabled once an image or audio chunk is present, and works again as soon as both prompts are text-only. Assisted-by: Claude Sonnet --- tools/server/server-common.cpp | 2 +- tools/server/server-common.h | 4 + tools/server/server-context.cpp | 17 +--- tools/server/tests/unit/test_slot_save.py | 119 ++++++++++++++++++++++ tools/server/tests/utils.py | 3 + 5 files changed, 132 insertions(+), 13 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index eade7db21256..b1140d55f913 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -644,7 +644,7 @@ llama_tokens server_tokens::get_text_tokens() const { } void server_tokens::set_token(llama_pos pos, llama_token id) { - GGML_ASSERT(!has_mtmd); // only allow this if mtmd is disabled + GGML_ASSERT(!has_mtmd || map_idx_to_media.empty()); // only allow this on text-only tokens tokens[pos] = id; } diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 9894f5f06fb0..b085fbe47f56 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -222,6 +222,10 @@ struct server_tokens { bool empty() const { return tokens.empty(); } + // true if the token list holds real media chunks + // note: this differs from has_mtmd, which only means an mmproj is loaded + bool has_media_chunks() const { return !map_idx_to_media.empty(); } + void clear() { map_idx_to_media.clear(); tokens.clear(); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index bc5fbf937fef..1902c6462190 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1211,10 +1211,7 @@ struct server_context_impl { SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled"); } - if (params_base.n_cache_reuse) { - params_base.n_cache_reuse = 0; - SRV_WRN("%s\n", "cache_reuse is not supported by multimodal, it will be disabled"); - } + // keep cache_reuse: it is applied per request and only while the prompt has no media } if (!llama_memory_can_shift(llama_get_memory(ctx_tgt))) { @@ -3419,9 +3416,12 @@ struct server_context_impl { const auto n_cache_reuse = slot.task->params.n_cache_reuse; + // cache reuse shifts KV cells around, which cannot cross a media chunk. + // with an mmproj loaded it still works as long as both prompts stay text-only. const bool can_cache_reuse = llama_memory_can_shift(llama_get_memory(ctx_tgt)) && - !slot.prompt.tokens.has_mtmd; + !slot.prompt.tokens.has_media_chunks() && + !input_tokens.has_media_chunks(); if (!can_cache_reuse && n_cache_reuse > 0) { SLT_WRN(slot, "cache reuse is not supported - ignoring n_cache_reuse = %d\n", n_cache_reuse); @@ -3429,16 +3429,9 @@ struct server_context_impl { // reuse chunks from the cached prompt by shifting their KV cache in the new position if (can_cache_reuse && n_cache_reuse > 0) { - GGML_ASSERT(!slot.prompt.tokens.has_mtmd); - size_t head_c = n_past; // cache size_t head_p = n_past; // current prompt - if (mctx) { - // we should never reach this - GGML_ABORT("not supported by multimodal"); - } - SLT_DBG(slot, "trying to reuse chunks with size > %d, n_past = %d\n", n_cache_reuse, n_past); while (head_c < slot.prompt.tokens.size() && diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py index 5eca46cb292d..cdb76daf074e 100644 --- a/tools/server/tests/unit/test_slot_save.py +++ b/tools/server/tests/unit/test_slot_save.py @@ -493,6 +493,125 @@ def test_slot_save_restore_image_payload_larger_than_context(mmproj_server): assert res.body["timings"]["prompt_n"] == 1 +# +# Prompt cache reuse on a multimodal server (mmproj loaded). +# +# Cache reuse is gated on real media chunks, not on has_mtmd. +# Text-only prompts must still reuse a shifted matching chunk while an mmproj is loaded. +# Reuse stays disabled while either the cached or the incoming prompt carries media. +# swa_full keeps the shifted match valid: the default SWA cache drops it on checkpoint validation. +# cache_ram 0 disables the RAM prompt cache so only the n_cache_reuse shift path can reuse tokens. +# + +CACHE_REUSE_LEAD = "Throw away this opening line." + +CACHE_REUSE_TEXT = ( + " Alpha beta gamma delta epsilon zeta eta theta iota kappa" + " lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega." +) + + +def test_cache_reuse_text_only_with_mmproj(mmproj_server): + server = mmproj_server + server.cache_reuse = 4 + server.swa_full = True + server.cache_ram = 0 + server.start() + + # prime the slot with a text-only prompt that has an extra leading sentence + res = server.make_request("POST", "/completion", data={ + "prompt": CACHE_REUSE_LEAD + CACHE_REUSE_TEXT, + "id_slot": 0, + "cache_prompt": True, + "n_predict": 1, + }) + assert res.status_code == 200 + + # resend without the leading sentence: the shared chunk must shift its KV cache and be reused + res = server.make_request("POST", "/completion", data={ + "prompt": CACHE_REUSE_TEXT, + "id_slot": 0, + "cache_prompt": True, + "n_predict": 1, + }) + assert res.status_code == 200 + cache_n = res.body["timings"]["cache_n"] + prompt_n = res.body["timings"]["prompt_n"] + assert cache_n > 10 # the matching chunk was shifted and reused + assert prompt_n < cache_n + + +def test_cache_reuse_disabled_when_media_present(mmproj_server): + server = mmproj_server + server.cache_reuse = 4 + server.swa_full = True + server.cache_ram = 0 + server.start() + + img = _get_img_base64(IMG_URL_CAT) + + # cached prompt: a shiftable text chunk followed by media. + # the chunk would be shifted and reused if the media gate were missing. + res = server.make_request("POST", "/completions", data={ + "id_slot": 0, + "cache_prompt": True, + "n_predict": 1, + "prompt": { + "prompt_string": CACHE_REUSE_LEAD + CACHE_REUSE_TEXT + " <__media__>", + "multimodal_data": [img], + }, + }) + assert res.status_code == 200 + + res = server.make_request("POST", "/completion", data={ + "prompt": CACHE_REUSE_TEXT, + "id_slot": 0, + "cache_prompt": True, + "n_predict": 1, + }) + assert res.status_code == 200 + assert res.body["timings"]["cache_n"] < 10 # media in the cached prompt blocks reuse + + # incoming prompt: the same shiftable text chunk followed by media. + res = server.make_request("POST", "/completion", data={ + "prompt": CACHE_REUSE_LEAD + CACHE_REUSE_TEXT, + "id_slot": 1, + "cache_prompt": True, + "n_predict": 1, + }) + assert res.status_code == 200 + + res = server.make_request("POST", "/completions", data={ + "id_slot": 1, + "cache_prompt": True, + "n_predict": 1, + "prompt": { + "prompt_string": CACHE_REUSE_TEXT + " <__media__>", + "multimodal_data": [img], + }, + }) + assert res.status_code == 200 + assert res.body["timings"]["cache_n"] < 10 # media in the incoming prompt blocks reuse + + # slot 0 no longer holds media: a later text-only prompt reuses its shifted chunk again + res = server.make_request("POST", "/completion", data={ + "prompt": CACHE_REUSE_LEAD + CACHE_REUSE_TEXT, + "id_slot": 0, + "cache_prompt": True, + "n_predict": 1, + }) + assert res.status_code == 200 + + res = server.make_request("POST", "/completion", data={ + "prompt": CACHE_REUSE_TEXT, + "id_slot": 0, + "cache_prompt": True, + "n_predict": 1, + }) + assert res.status_code == 200 + assert res.body["timings"]["cache_n"] > 10 # reuse resumes once the media is gone + + def test_slot_restore_media_file_without_mmproj(mmproj_server): server = mmproj_server server.start() diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index 826aef2d5bcb..d245f3bd6437 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -113,6 +113,7 @@ class ServerProcess: media_path: str | None = None sleep_idle_seconds: int | None = None cache_ram: int | None = None + cache_reuse: int | None = None no_cache_idle_slots: bool = False log_path: str | None = None ui_mcp_proxy: bool = False @@ -278,6 +279,8 @@ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None: server_args.extend(["--sleep-idle-seconds", self.sleep_idle_seconds]) if self.cache_ram is not None: server_args.extend(["--cache-ram", self.cache_ram]) + if self.cache_reuse is not None: + server_args.extend(["--cache-reuse", self.cache_reuse]) if self.no_cache_idle_slots: server_args.append("--no-cache-idle-slots") if self.ui_mcp_proxy: From 6f38cce8d4ef2afe5674e9d1fcd15490d9e080ae Mon Sep 17 00:00:00 2001 From: piggidragon Date: Sun, 6 Sep 2026 00:43:03 +0200 Subject: [PATCH 2/4] server : warn about unsupported cache reuse only Cache reuse is no longer disabled at startup for mmproj, so the per-slot warning now fires on every request that carries media. Warn only when the memory cannot shift, and log the expected media case at debug level. Also simplify the set_token assert: a non-empty media map already implies has_mtmd. Assisted-by: Claude Opus 5 --- tools/server/server-common.cpp | 2 +- tools/server/server-context.cpp | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index b1140d55f913..beddeed05eaf 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -644,7 +644,7 @@ llama_tokens server_tokens::get_text_tokens() const { } void server_tokens::set_token(llama_pos pos, llama_token id) { - GGML_ASSERT(!has_mtmd || map_idx_to_media.empty()); // only allow this on text-only tokens + GGML_ASSERT(!has_media_chunks()); // only allow this on text-only tokens tokens[pos] = id; } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 1902c6462190..12f7690eeea2 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3416,15 +3416,21 @@ struct server_context_impl { const auto n_cache_reuse = slot.task->params.n_cache_reuse; + const bool can_shift = llama_memory_can_shift(llama_get_memory(ctx_tgt)); + // cache reuse shifts KV cells around, which cannot cross a media chunk. // with an mmproj loaded it still works as long as both prompts stay text-only. - const bool can_cache_reuse = - llama_memory_can_shift(llama_get_memory(ctx_tgt)) && - !slot.prompt.tokens.has_media_chunks() && - !input_tokens.has_media_chunks(); + const bool has_media = slot.prompt.tokens.has_media_chunks() || input_tokens.has_media_chunks(); + + const bool can_cache_reuse = can_shift && !has_media; - if (!can_cache_reuse && n_cache_reuse > 0) { - SLT_WRN(slot, "cache reuse is not supported - ignoring n_cache_reuse = %d\n", n_cache_reuse); + if (n_cache_reuse > 0) { + if (!can_shift) { + SLT_WRN(slot, "cache reuse is not supported - ignoring n_cache_reuse = %d\n", n_cache_reuse); + } else if (has_media) { + // expected on every request that carries media, so keep it out of the log + SLT_DBG(slot, "cache reuse is disabled while the prompt has media - ignoring n_cache_reuse = %d\n", n_cache_reuse); + } } // reuse chunks from the cached prompt by shifting their KV cache in the new position From dc5fc2707d4259c1253eb5cf3db0ff16dd20e1e7 Mon Sep 17 00:00:00 2001 From: piggidragon Date: Mon, 7 Sep 2026 11:44:18 +0200 Subject: [PATCH 3/4] server : fix cache reuse shift hazards Review follow-ups on the cache reuse block: - the shift is applied to the draft context too, so require both contexts to support it - skip reuse while an alora is invoked, the loop moved n_past past the cap - drop the context checkpoints after a shift, they no longer match the cache - set_token asserts on the media placeholder instead of the media map, and takes a token index, which is what the only caller passes Assisted-by: Claude Opus 5 --- tools/server/server-common.cpp | 7 ++++--- tools/server/server-common.h | 4 ++-- tools/server/server-context.cpp | 32 +++++++++++++++++++++++--------- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index beddeed05eaf..710632f76f8f 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -643,9 +643,10 @@ llama_tokens server_tokens::get_text_tokens() const { return res; } -void server_tokens::set_token(llama_pos pos, llama_token id) { - GGML_ASSERT(!has_media_chunks()); // only allow this on text-only tokens - tokens[pos] = id; +void server_tokens::set_token(size_t idx, llama_token id) { + // a media placeholder must stay in sync with the media map, so never write over one + GGML_ASSERT(tokens[idx] != LLAMA_TOKEN_NULL && id != LLAMA_TOKEN_NULL); + tokens[idx] = id; } void server_tokens::keep_first(size_t n) { diff --git a/tools/server/server-common.h b/tools/server/server-common.h index b085fbe47f56..bd429e995057 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -215,8 +215,8 @@ struct server_tokens { std::vector serialize() const; static server_tokens deserialize(const llama_tokens & packed, bool has_mtmd); - // for compatibility with speculative decoding - void set_token(llama_pos pos, llama_token id); + // overwrite a text token, media placeholders are not writable + void set_token(size_t idx, llama_token id); size_t size() const { return tokens.size(); } diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 12f7690eeea2..4006098e82cc 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3416,28 +3416,35 @@ struct server_context_impl { const auto n_cache_reuse = slot.task->params.n_cache_reuse; - const bool can_shift = llama_memory_can_shift(llama_get_memory(ctx_tgt)); + // the shift is applied to the draft context too, so both must support it + const bool can_shift = llama_memory_can_shift(llama_get_memory(ctx_tgt)) && + (!ctx_dft || llama_memory_can_shift(llama_get_memory(ctx_dft))); - // cache reuse shifts KV cells around, which cannot cross a media chunk. - // with an mmproj loaded it still works as long as both prompts stay text-only. + // the loop below uses token indices as positions, which a media chunk breaks. + // an mmproj alone is fine, only a real media chunk in either prompt is not const bool has_media = slot.prompt.tokens.has_media_chunks() || input_tokens.has_media_chunks(); - const bool can_cache_reuse = can_shift && !has_media; + // the loop moves n_past past the alora cap applied above + const bool has_alora = slot.alora_invocation_start > 0; - if (n_cache_reuse > 0) { + const bool can_cache_reuse = n_cache_reuse > 0 && can_shift && !has_media && !has_alora; + + if (n_cache_reuse > 0 && !can_cache_reuse) { if (!can_shift) { SLT_WRN(slot, "cache reuse is not supported - ignoring n_cache_reuse = %d\n", n_cache_reuse); - } else if (has_media) { - // expected on every request that carries media, so keep it out of the log - SLT_DBG(slot, "cache reuse is disabled while the prompt has media - ignoring n_cache_reuse = %d\n", n_cache_reuse); + } else { + // expected on every request with media or an alora, so keep it out of the log + SLT_DBG(slot, "cache reuse is disabled for this prompt - ignoring n_cache_reuse = %d\n", n_cache_reuse); } } // reuse chunks from the cached prompt by shifting their KV cache in the new position - if (can_cache_reuse && n_cache_reuse > 0) { + if (can_cache_reuse) { size_t head_c = n_past; // cache size_t head_p = n_past; // current prompt + bool kv_shifted = false; + SLT_DBG(slot, "trying to reuse chunks with size > %d, n_past = %d\n", n_cache_reuse, n_past); while (head_c < slot.prompt.tokens.size() && @@ -3461,6 +3468,8 @@ struct server_context_impl { slot.mem.seq_rm (slot.id, head_p, head_c); slot.mem.seq_add(slot.id, head_c, head_c + n_match, kv_shift); + kv_shifted |= kv_shift != 0; + for (size_t i = 0; i < n_match; i++) { slot.prompt.tokens.set_token(head_p + i, slot.prompt.tokens[head_c + i]); n_past++; @@ -3473,6 +3482,11 @@ struct server_context_impl { } } + if (kv_shifted) { + // the checkpoints were taken before the shift, they no longer match the cache + slot.prompt.checkpoints.clear(); + } + SLT_DBG(slot, "after context reuse, new n_past = %d\n", n_past); } } else { From 43ecb2020a5f31e436ce55d5c16aedb57a28c6f7 Mon Sep 17 00:00:00 2001 From: piggidragon Date: Mon, 7 Sep 2026 11:44:18 +0200 Subject: [PATCH 4/4] server : merge the cache reuse tests One server instead of two. The shared chunk now starts on a newline, so it tokenizes the same with and without the leading sentence. Assisted-by: Claude Opus 5 --- tools/server/tests/unit/test_slot_save.py | 138 +++++++--------------- 1 file changed, 42 insertions(+), 96 deletions(-) diff --git a/tools/server/tests/unit/test_slot_save.py b/tools/server/tests/unit/test_slot_save.py index cdb76daf074e..e5885dfcec09 100644 --- a/tools/server/tests/unit/test_slot_save.py +++ b/tools/server/tests/unit/test_slot_save.py @@ -497,51 +497,19 @@ def test_slot_save_restore_image_payload_larger_than_context(mmproj_server): # Prompt cache reuse on a multimodal server (mmproj loaded). # # Cache reuse is gated on real media chunks, not on has_mtmd. -# Text-only prompts must still reuse a shifted matching chunk while an mmproj is loaded. -# Reuse stays disabled while either the cached or the incoming prompt carries media. -# swa_full keeps the shifted match valid: the default SWA cache drops it on checkpoint validation. -# cache_ram 0 disables the RAM prompt cache so only the n_cache_reuse shift path can reuse tokens. +# swa_full keeps the shifted match valid, cache_ram 0 leaves the KV shift as the only reuse path. # CACHE_REUSE_LEAD = "Throw away this opening line." +# starts on a newline, so the shared chunk tokenizes the same with and without the lead CACHE_REUSE_TEXT = ( - " Alpha beta gamma delta epsilon zeta eta theta iota kappa" + "\nAlpha beta gamma delta epsilon zeta eta theta iota kappa" " lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega." ) -def test_cache_reuse_text_only_with_mmproj(mmproj_server): - server = mmproj_server - server.cache_reuse = 4 - server.swa_full = True - server.cache_ram = 0 - server.start() - - # prime the slot with a text-only prompt that has an extra leading sentence - res = server.make_request("POST", "/completion", data={ - "prompt": CACHE_REUSE_LEAD + CACHE_REUSE_TEXT, - "id_slot": 0, - "cache_prompt": True, - "n_predict": 1, - }) - assert res.status_code == 200 - - # resend without the leading sentence: the shared chunk must shift its KV cache and be reused - res = server.make_request("POST", "/completion", data={ - "prompt": CACHE_REUSE_TEXT, - "id_slot": 0, - "cache_prompt": True, - "n_predict": 1, - }) - assert res.status_code == 200 - cache_n = res.body["timings"]["cache_n"] - prompt_n = res.body["timings"]["prompt_n"] - assert cache_n > 10 # the matching chunk was shifted and reused - assert prompt_n < cache_n - - -def test_cache_reuse_disabled_when_media_present(mmproj_server): +def test_cache_reuse_with_mmproj(mmproj_server): server = mmproj_server server.cache_reuse = 4 server.swa_full = True @@ -550,66 +518,44 @@ def test_cache_reuse_disabled_when_media_present(mmproj_server): img = _get_img_base64(IMG_URL_CAT) - # cached prompt: a shiftable text chunk followed by media. - # the chunk would be shifted and reused if the media gate were missing. - res = server.make_request("POST", "/completions", data={ - "id_slot": 0, - "cache_prompt": True, - "n_predict": 1, - "prompt": { - "prompt_string": CACHE_REUSE_LEAD + CACHE_REUSE_TEXT + " <__media__>", - "multimodal_data": [img], - }, - }) - assert res.status_code == 200 - - res = server.make_request("POST", "/completion", data={ - "prompt": CACHE_REUSE_TEXT, - "id_slot": 0, - "cache_prompt": True, - "n_predict": 1, - }) - assert res.status_code == 200 - assert res.body["timings"]["cache_n"] < 10 # media in the cached prompt blocks reuse - - # incoming prompt: the same shiftable text chunk followed by media. - res = server.make_request("POST", "/completion", data={ - "prompt": CACHE_REUSE_LEAD + CACHE_REUSE_TEXT, - "id_slot": 1, - "cache_prompt": True, - "n_predict": 1, - }) - assert res.status_code == 200 - - res = server.make_request("POST", "/completions", data={ - "id_slot": 1, - "cache_prompt": True, - "n_predict": 1, - "prompt": { - "prompt_string": CACHE_REUSE_TEXT + " <__media__>", - "multimodal_data": [img], - }, - }) - assert res.status_code == 200 - assert res.body["timings"]["cache_n"] < 10 # media in the incoming prompt blocks reuse - - # slot 0 no longer holds media: a later text-only prompt reuses its shifted chunk again - res = server.make_request("POST", "/completion", data={ - "prompt": CACHE_REUSE_LEAD + CACHE_REUSE_TEXT, - "id_slot": 0, - "cache_prompt": True, - "n_predict": 1, - }) - assert res.status_code == 200 - - res = server.make_request("POST", "/completion", data={ - "prompt": CACHE_REUSE_TEXT, - "id_slot": 0, - "cache_prompt": True, - "n_predict": 1, - }) - assert res.status_code == 200 - assert res.body["timings"]["cache_n"] > 10 # reuse resumes once the media is gone + def send_text(prompt, id_slot): + res = server.make_request("POST", "/completion", data={ + "prompt": prompt, + "id_slot": id_slot, + "cache_prompt": True, + "n_predict": 1, + }) + assert res.status_code == 200 + return res.body["timings"]["cache_n"] + + def send_media(prompt_string, id_slot): + res = server.make_request("POST", "/completions", data={ + "id_slot": id_slot, + "cache_prompt": True, + "n_predict": 1, + "prompt": { + "prompt_string": prompt_string, + "multimodal_data": [img], + }, + }) + assert res.status_code == 200 + return res.body["timings"]["cache_n"] + + # text-only: dropping the lead must shift the shared chunk and reuse it + send_text(CACHE_REUSE_LEAD + CACHE_REUSE_TEXT, 0) + assert send_text(CACHE_REUSE_TEXT, 0) > 10 + + # media in the cached prompt blocks the very same shift + send_media(CACHE_REUSE_LEAD + CACHE_REUSE_TEXT + " <__media__>", 1) + assert send_text(CACHE_REUSE_TEXT, 1) < 10 + + # media in the incoming prompt blocks it too, even though it sits after the shared chunk + send_text(CACHE_REUSE_LEAD + CACHE_REUSE_TEXT, 1) + assert send_media(CACHE_REUSE_TEXT + " <__media__>", 1) < 10 + + # reuse resumes as soon as the slot holds text only again + send_text(CACHE_REUSE_LEAD + CACHE_REUSE_TEXT, 1) + assert send_text(CACHE_REUSE_TEXT, 1) > 10 def test_slot_restore_media_file_without_mmproj(mmproj_server):