From bf2826ec3613646f5237cc7d52ef8669e9f47f4a Mon Sep 17 00:00:00 2001 From: mrciffa Date: Wed, 23 Sep 2026 12:37:17 +0200 Subject: [PATCH 1/3] feat(qwen35): speculative decoding for image requests Image tokens take 2D rotary positions, so after an image the rotary position runs rope_delta_ ahead of the KV position. AR decode already applied that offset; the DFlash verify target did not, so the HTTP layer and the backend forced every image request to plain AR decode. Qwen35DFlashTarget now reads the backend's per-request rope_delta_ and shifts the M-RoPE positions of chain and tree verify by it (zero for text, so text requests are unchanged). The blanket AR force for images is dropped from the HTTP layer and the Qwen3.5 backend; DeepSeek4 keeps its own image guard and still decodes image requests AR. R9700, Qwen3.8-27B-IQ4_XS-pure + Q8_0 projector + DFlash2, 12 images, 256-token answers: 4.03 s per answer (76 tok/s) vs 7.58 s (36 tok/s) before; llama.cpp with the same drafter 5.54 s, without 8.36 s. Image eval 188/220 unchanged (218 answers identical), 1-4 images 9/9. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/image-input.md | 21 +++++++++++++-------- server/src/qwen35/qwen35_backend.cpp | 7 ++++--- server/src/qwen35/qwen35_dflash_target.cpp | 4 ++-- server/src/qwen35/qwen35_dflash_target.h | 7 +++++++ server/src/server/http_server.cpp | 3 ++- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/docs/image-input.md b/docs/image-input.md index d4a264b9d..eb73ba092 100644 --- a/docs/image-input.md +++ b/docs/image-input.md @@ -36,8 +36,8 @@ hf download Lucebox/Qwen3.8-27B-DFlash2-GGUF \ --port 8216 ``` -About 21 GiB of VRAM at the peak of an image request. Text requests keep the -DFlash2 drafter; image requests decode without it. +About 21 GiB of VRAM at the peak of an image request. Text and image requests +both decode with the DFlash2 drafter. ### DeepSeek V4 Flash Vision on a Strix Halo @@ -106,10 +106,11 @@ Decoder pixel and aspect limits also apply. A model's image marker cannot be sup as ordinary text. The server expands image markers after final rendering and tokenization, and -the expanded image tokens count toward context and usage. Image requests use -plain autoregressive decoding and bypass the token-keyed prefix, disk and -agent-turn caches and prompt compression: tokens alone do not identify an -image. Text requests on the same server keep speculative decoding and caching. +the expanded image tokens count toward context and usage. Image requests +bypass the token-keyed prefix, disk and agent-turn caches and prompt +compression: tokens alone do not identify an image. Qwen3.5 / Qwen3.8 image +requests decode with the drafter like text; DS4V image requests decode +without it. Layer or tensor splitting across GPUs, remote target shards, concurrent sequence scheduling (`--max-concurrency`) and upstream forwarding do not @@ -144,8 +145,12 @@ Lucebox `Qwen3.8-27B-IQ4_XS-pure` file and a Q8_0 projector: lmms-eval prompts: AI2D 90/100, ChartQA relaxed accuracy 56/60 (augmented) and 42/60 (human). Image prompts prefill in 0.56 s on average; one to four images per request all answer correctly (four images, 2,495 tokens: 3.2 s). -- Text decodes at 56 to 117 tok/s on 256-token answers (84 on average); image - requests decode without the drafter at about 36 tok/s. +- Text decodes at 56 to 117 tok/s on 256-token answers (84 on average). +- Image requests decode with the drafter. On 12 images with 256-token + answers: 4.0 s per answer (76 tok/s after the first token), against 5.5 s + for llama.cpp with the same drafter (`--spec-type draft-dflash`) and 8.4 s + without one; faster on every image, 1.21x to 1.58x. The 220-question score + is unchanged (188, 218 answers identical to plain decode). With unsloth's UD-IQ4_XS file and the published BF16 projector: diff --git a/server/src/qwen35/qwen35_backend.cpp b/server/src/qwen35/qwen35_backend.cpp index 2dbbaa4dd..6d8ddf3ea 100644 --- a/server/src/qwen35/qwen35_backend.cpp +++ b/server/src/qwen35/qwen35_backend.cpp @@ -1324,6 +1324,7 @@ DFlashTarget * Qwen35Backend::dflash_target() { qt->set_kvflash_pager(&kvflash_pager_); } qt->set_fast_rollback(cfg_.fast_rollback); + qt->set_rope_offset(&rope_delta_); } return dflash_target_.get(); } @@ -1508,9 +1509,9 @@ GenerateResult Qwen35Backend::generate_impl(const GenerateRequest & req, req.n_gen, ar_n_gen, committed, cfg_.device.max_ctx); } } - // Speculative decoding takes rotary positions from KV positions, which - // an image prompt pulls apart, so image requests decode one by one. - if (cfg_.paged_attention || req.force_ar_decode || has_images) { + // Image requests speculate too: the verify target shifts its rotary + // positions by rope_delta_, and the drafter only proposes tokens. + if (cfg_.paged_attention || req.force_ar_decode) { decode_ok = do_ar_decode(committed, ar_n_gen, result.tokens, out_io, req.budget_hook, &result.budget_forced_close, diff --git a/server/src/qwen35/qwen35_dflash_target.cpp b/server/src/qwen35/qwen35_dflash_target.cpp index 8153bc0e0..478ee0d02 100644 --- a/server/src/qwen35/qwen35_dflash_target.cpp +++ b/server/src/qwen35/qwen35_dflash_target.cpp @@ -307,7 +307,7 @@ bool Qwen35DFlashTarget::verify_batch( // GGML M-RoPE positions are axis-major. std::vector pos(4 * n_tokens); - fill_qwen35_mrope_positions(pos.data(), base_pos, n_tokens); + fill_qwen35_mrope_positions(pos.data(), base_pos + rope_offset(), n_tokens); ggml_backend_tensor_set(sg_.positions, pos.data(), 0, sizeof(int32_t) * pos.size()); @@ -452,7 +452,7 @@ bool Qwen35DFlashTarget::verify_tree( // M-RoPE axis-major positions: each node sits at committed + its depth. std::vector pos4(4 * N, 0); for (int i = 0; i < N_actual; i++) { - const int p = committed + (i == 0 ? 0 : tree.depths[i - 1]); + const int p = committed + rope_offset() + (i == 0 ? 0 : tree.depths[i - 1]); pos4[0 * N + i] = p; pos4[1 * N + i] = p; pos4[2 * N + i] = p; diff --git a/server/src/qwen35/qwen35_dflash_target.h b/server/src/qwen35/qwen35_dflash_target.h index e782d262a..425545b89 100644 --- a/server/src/qwen35/qwen35_dflash_target.h +++ b/server/src/qwen35/qwen35_dflash_target.h @@ -93,6 +93,11 @@ class Qwen35DFlashTarget : public DFlashTarget { // so rollback_to() can restore recurrent state without replay. void set_fast_rollback(bool enabled) { fast_rollback_ = enabled; } + // Rotary positions run ahead of KV positions by this many after an image + // (image tokens take 2D positions). Points at the owner's per-request + // offset, which every prefill sets; null means zero. + void set_rope_offset(const int * offset) { rope_offset_ = offset; } + private: TargetWeights & w_; TargetCache & cache_; @@ -102,6 +107,8 @@ class Qwen35DFlashTarget : public DFlashTarget { int fa_window_; KvFlashPager * pager_ = nullptr; bool fast_rollback_ = false; + const int * rope_offset_ = nullptr; + int rope_offset() const { return rope_offset_ ? *rope_offset_ : 0; } // SpecLA (docs/SPECLA.md): true when the cache was // migrated with factor buffers. Capture-verify then runs the diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 18d8c7973..655d182c2 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -4102,7 +4102,8 @@ void HttpServer::prepare_generation_inputs( inputs.request.prompt = prepared.tokens; inputs.request.images = prepared.images; - inputs.request.force_ar_decode = bool(prepared.images); + // Image requests may speculate: each backend decides (DeepSeek4 decodes + // them one by one, Qwen3.5 verifies at image-shifted rotary positions). inputs.request.n_gen = inputs.generation_cap; inputs.request.sampler = req.sampler; inputs.request.do_sample = req.sampler.needs_logit_processing(); From 04ce6a1bd3bc810e8b32cb40a64a360703aba2e6 Mon Sep 17 00:00:00 2001 From: mrciffa Date: Wed, 23 Sep 2026 13:18:30 +0200 Subject: [PATCH 2/3] feat(deepseek4): DSpark speculative decoding for image requests The image prefill graph takes no DSpark capture hooks, so image requests skipped feature capture and decoded AR. Image chunks now end at their last image, the text after the image prefills and captures as ordinary chunks, and the feature window is cleared at each image chunk so the drafter always reads one contiguous tail. With that, image requests take the DSpark path like text. Strix Halo alone, Vision-Exp ROCMFPX MIX, published DSpark launch, 12 images, 256-token answers: 13.3 s per answer (30.4 tok/s) vs 15.7 s (22.1 tok/s); first token about 0.7 s later from the capture band. 220 image questions 181 (AI2D 86, ChartQA 55/40), 209 identical to plain decode; one to four images 9/9; text decode unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/image-input.md | 14 +++++++---- server/src/deepseek4/deepseek4_backend.cpp | 28 ++++++++++++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/docs/image-input.md b/docs/image-input.md index eb73ba092..614d04a4a 100644 --- a/docs/image-input.md +++ b/docs/image-input.md @@ -108,9 +108,8 @@ as ordinary text. The server expands image markers after final rendering and tokenization, and the expanded image tokens count toward context and usage. Image requests bypass the token-keyed prefix, disk and agent-turn caches and prompt -compression: tokens alone do not identify an image. Qwen3.5 / Qwen3.8 image -requests decode with the drafter like text; DS4V image requests decode -without it. +compression: tokens alone do not identify an image. Image requests decode +with the model's drafter like text requests. Layer or tensor splitting across GPUs, remote target shards, concurrent sequence scheduling (`--max-concurrency`) and upstream forwarding do not @@ -265,7 +264,14 @@ importance matrix, the shipped recipe above), on a Strix Halo alone at top-k 6: and one-to-four-image sets are all correct. - With the published DSpark drafter and fused decode and verify, text decodes at 25 to 37 tok/s on 256-token answers (30 mean), as fast as the shipped - text model; image requests decode without the drafter at about 22 tok/s. + text model. +- Image requests decode with the DSpark drafter too: on 12 images with + 256-token answers, 13.3 s per answer (30 tok/s after the first token) + against 15.7 s (22 tok/s) without it. Capturing the drafter's features + during prefill adds about 0.7 s before the first token, so one-word answers + come back slightly later. The 220 questions score AI2D 86, ChartQA 55 and + 40 with the drafter (209 answers identical to plain decode); one to four + images all correct. Not yet established: diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index d45d9b762..a4d3e3274 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -2641,8 +2641,9 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, int snap_slot, int snap_pos, const DeepSeek4ImagePrompt * images) { - const bool capture_spec = !images && spec_enabled_ && spec_drafter_; - if (images) spec_feat_window_.clear(); + // Image prompts capture DSpark features from their text chunks only: the + // image graph takes no capture hooks (see the chunking below). + const bool capture_spec = spec_enabled_ && spec_drafter_; const InferencePhase phase = deepseek4_roctx_prefill_phase( prefill_attention_mode_name(cfg_.prefill_mode)); const DeepSeek4RoctxPhaseScope roctx_phase(phase); @@ -2823,10 +2824,29 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, spec_snap_from, spec_snap_to); } + bool chunk_has_image = false; if (images) { n_tok = vision::atomic_image_chunk(images->spans(), uint64_t(pos), n_tok, uint64_t(n_total - i), image_capacity); if (!n_tok) return -1; + // An image batch cannot capture DSpark features, so end it at its + // last image: the text after the image then prefills (and + // captures) as ordinary chunks. + const vision::ImageSpanView spans = images->spans(); + uint64_t last_image_end = 0; + for (size_t k = 0; k < spans.size; ++k) { + const auto & span = spans.data[k]; + if (span.block_begin < uint64_t(pos + n_tok) && span.block_end > uint64_t(pos)) { + chunk_has_image = true; + last_image_end = std::max(last_image_end, span.block_end); + } + } + if (chunk_has_image && capture_spec && last_image_end < uint64_t(pos + n_tok)) { + n_tok = int(last_image_end - uint64_t(pos)); + } + // The drafter reads the newest rows as one contiguous window, so + // rows from before an image cannot sit next to rows after it. + if (chunk_has_image) spec_feat_window_.clear(); } // Bulk prompt graphs and the final DSpark feature-capture graph have @@ -2873,7 +2893,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, const bool capture_snapshot = !snapshot_saved && i < spec_snap_to && i + n_tok > spec_snap_from; - if (capture_spec && + if (capture_spec && !chunk_has_image && (capture_final || capture_snapshot)) { spec_hooks.capture_layer_ids = &spec_drafter_->capture_layer_ids; spec_hooks.capture_out = &spec_cap; @@ -3279,7 +3299,7 @@ GenerateResult DeepSeek4Backend::generate_from_state( } } if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && - !req.images && !req.force_ar_decode && !budget_requires_ar && !sampling_requires_ar) { + !req.force_ar_decode && !budget_requires_ar && !sampling_requires_ar) { if (last_logits_.empty()) { result.fail(GenerateErrorCode::DecodeFailed, "spec: no prefill logits"); return result; From c2b02a3974a1852290cab91c33197f503bd652eb Mon Sep 17 00:00:00 2001 From: mrciffa Date: Thu, 24 Sep 2026 13:31:02 +0200 Subject: [PATCH 3/3] fix(vision): address review on image speculative decode - DeepSeek4 falls back to plain decode when an image request has no text after the last image to seed the drafter window. - vision::last_image_end_in replaces the inline span loop in do_prefill, with unit checks. - Clarify the http_server comment and the image-input doc figures. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/image-input.md | 12 ++++++++---- server/src/common/vision/image_spans.h | 11 +++++++++++ server/src/deepseek4/deepseek4_backend.cpp | 17 +++++++---------- server/src/server/http_server.cpp | 5 +++-- server/test/test_ds4v_image_integration.cpp | 5 +++++ 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/docs/image-input.md b/docs/image-input.md index 614d04a4a..930df908b 100644 --- a/docs/image-input.md +++ b/docs/image-input.md @@ -148,7 +148,8 @@ Lucebox `Qwen3.8-27B-IQ4_XS-pure` file and a Q8_0 projector: - Image requests decode with the drafter. On 12 images with 256-token answers: 4.0 s per answer (76 tok/s after the first token), against 5.5 s for llama.cpp with the same drafter (`--spec-type draft-dflash`) and 8.4 s - without one; faster on every image, 1.21x to 1.58x. The 220-question score + without one; faster than llama.cpp with the drafter on every image, 1.21x + to 1.58x. The 220-question score is unchanged (188, 218 answers identical to plain decode). With unsloth's UD-IQ4_XS file and the published BF16 projector: @@ -156,7 +157,8 @@ With unsloth's UD-IQ4_XS file and the published BF16 projector: - 220 seeded questions from `lmms-lab/ai2d` and `lmms-lab/ChartQA` with lmms-eval prompts: AI2D 85/100, ChartQA relaxed accuracy 55/60 (augmented) and 43/60 (human), no errors. Image prompts average 448 tokens and prefill in - 0.71 s (largest 1,068 tokens, 1.8 s); decode runs at 31 to 35 tok/s. + 0.71 s (largest 1,068 tokens, 1.8 s); decode runs at 31 to 35 tok/s (plain + decode, measured before image requests used the drafter). - A projector with its weight matrices in Q8_0 (rows that are not a multiple of 32 stay F16) encodes a 975-token image in 443 ms instead of 677 ms with the BF16 file, with the same scores on the 220 questions and 216 identical @@ -171,7 +173,8 @@ With unsloth's UD-IQ4_XS file and the published BF16 projector: tokens). The projector adds 0.9 GiB of VRAM; the peak during image requests was 21.6 GiB against 20.8 GiB for text. - The same requests answer correctly on a Strix Halo alone, where a - 1,012-token image prompt prefills in 4.6 s and decodes at 14 tok/s. + 1,012-token image prompt prefills in 4.6 s and decodes at 14 tok/s (plain + decode). Not yet established: a comparison against the reference implementation on the same questions, and CUDA. The tower uses only standard ggml @@ -250,7 +253,8 @@ the exported projector, on a Strix Halo alone and on R9700 + Strix Halo, 220 seeded questions from `lmms-lab/ai2d` and `lmms-lab/ChartQA` with lmms-eval prompts: AI2D 85/100, ChartQA relaxed accuracy 55/60 (augmented) and 43/60 (human). Both layouts score the same and give word-identical answers on 213 of -220 questions. An image request prefills in about 4 s and decodes at about +220 questions. An image request prefills in about 4 s and, without the +drafter, decodes at about 23 tok/s. With our own ROCMFP MIX conversion of the same checkpoint (per-expert diff --git a/server/src/common/vision/image_spans.h b/server/src/common/vision/image_spans.h index bf1c14a82..2ae2f9648 100644 --- a/server/src/common/vision/image_spans.h +++ b/server/src/common/vision/image_spans.h @@ -33,6 +33,17 @@ inline const TokenSpan * image_block_at(ImageSpanView spans, uint64_t position) return nullptr; } +// End of the last image block that overlaps [begin, end), or 0 when none does. +inline uint64_t last_image_end_in(ImageSpanView spans, uint64_t begin, uint64_t end) { + uint64_t last = 0; + for (size_t i = 0; i < spans.size; ++i) { + const auto & span = spans.data[i]; + if (span.block_begin >= end) break; + if (span.block_end > begin) last = span.block_end; + } + return last; +} + inline bool valid_image_spans(ImageSpanView spans, uint64_t prompt_size, size_t max_images, uint64_t max_block_tokens) { if (spans.size > max_images || (spans.size && !spans.data)) return false; diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index a4d3e3274..b07ce6961 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -2832,15 +2832,9 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, // An image batch cannot capture DSpark features, so end it at its // last image: the text after the image then prefills (and // captures) as ordinary chunks. - const vision::ImageSpanView spans = images->spans(); - uint64_t last_image_end = 0; - for (size_t k = 0; k < spans.size; ++k) { - const auto & span = spans.data[k]; - if (span.block_begin < uint64_t(pos + n_tok) && span.block_end > uint64_t(pos)) { - chunk_has_image = true; - last_image_end = std::max(last_image_end, span.block_end); - } - } + const uint64_t last_image_end = + vision::last_image_end_in(images->spans(), uint64_t(pos), uint64_t(pos + n_tok)); + chunk_has_image = last_image_end != 0; if (chunk_has_image && capture_spec && last_image_end < uint64_t(pos + n_tok)) { n_tok = int(last_image_end - uint64_t(pos)); } @@ -3298,7 +3292,10 @@ GenerateResult DeepSeek4Backend::generate_from_state( sampler_.rep_pen, sampler_.freq_pen, sampler_.pres_pen); } } - if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && + // An image prompt whose last image leaves no captured text rows gives the + // drafter no context to start from; decode that request plainly. + const bool image_without_draft_context = req.images && spec_feat_window_.empty(); + if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && !image_without_draft_context && !req.force_ar_decode && !budget_requires_ar && !sampling_requires_ar) { if (last_logits_.empty()) { result.fail(GenerateErrorCode::DecodeFailed, "spec: no prefill logits"); diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index 655d182c2..7dbd4be54 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -4102,8 +4102,9 @@ void HttpServer::prepare_generation_inputs( inputs.request.prompt = prepared.tokens; inputs.request.images = prepared.images; - // Image requests may speculate: each backend decides (DeepSeek4 decodes - // them one by one, Qwen3.5 verifies at image-shifted rotary positions). + // Image requests may speculate; each backend decides (Qwen3.5 verifies at + // image-shifted rotary positions, DeepSeek4 drafts from the text after + // the last image). inputs.request.n_gen = inputs.generation_cap; inputs.request.sampler = req.sampler; inputs.request.do_sample = req.sampler.needs_logit_processing(); diff --git a/server/test/test_ds4v_image_integration.cpp b/server/test/test_ds4v_image_integration.cpp index 45cb7967f..84837cc35 100644 --- a/server/test/test_ds4v_image_integration.cpp +++ b/server/test/test_ds4v_image_integration.cpp @@ -72,6 +72,11 @@ void validation_and_lookup() { std::array too_many{}; require(!valid_image_spans({too_many.data(), too_many.size()}, 100), "more than four images rejected"); const std::vector spans{{10, 13, 18, 20}, {20, 20, 25, 25}, {30, 31, 33, 35}}; + require(last_image_end_in(view(spans), 0, 10) == 0, "no image before the first block"); + require(last_image_end_in(view(spans), 0, 11) == 20, "chunk reaching into the first image"); + require(last_image_end_in(view(spans), 12, 26) == 25, "last overlapping image wins"); + require(last_image_end_in(view(spans), 25, 30) == 0, "text between images"); + require(last_image_end_in(view(spans), 34, 40) == 35, "chunk starting inside an image"); require(valid_image_spans(view(spans), 35), "adjacent and separated blocks valid"); require(!image_block_at(view(spans), 9), "text before block excluded"); require(image_block_at(view(spans), 10) == &spans[0], "leading padding belongs to image block");