fix(deepseek4): make batched image serving safe for many users

docs/image-input.md

@@ -150,16 +150,29 @@ luce_server models/DeepSeek-V4-Flash-Vision-Exp-ROCMFPX-MIX-STRIX.gguf \
 ```
 
 Its image blocks need whole-block bidirectional prefill, which the batched
-engine's 16-row step cannot run. Image requests admitted since the last step
-are therefore prefilled up to their last token together, in shared
-layer-major sparse passes into per-request staging caches (each layer's
-experts are read once for all of them); that state is copied into each
-request's paged slot and the last token prefills in the batch, so the answers
-decode alongside everyone else. On the Strix Halo with the encoder on the
-R9700, four concurrent image answers of 256 tokens finish in 35 s (29 tok/s in
-total), two images plus two text requests at 31 tok/s; four text requests
-reach 38 tok/s. Image requests beyond the free slots wait in the queue. `/props` reports the
-effective capability in `capabilities.image_input_supported` after backend
+engine's 16-row step cannot run. An image request is therefore prefilled up to
+its last token into its slot's staging cache on the layer-major sparse path,
+then copied into its paged slot, and the last token prefills in the batch, so
+the answer decodes alongside everyone else. Every layer must see a whole image
+block at once, so the block cannot be split by rows; it is split by layers
+instead. While other requests are decoding, a staged pass takes about 256 rows
+(a whole image block, which may be more) shared by the pending image requests
+and runs 6 of its 43 layers per batched step, so live streams decode after
+every slice. The result is identical to running the pass in one go. With
+nothing decoding, a pass takes up to 1,024 rows and all its layers at once, so
+the requests share each layer's expert reads. With `--mmproj-device`, the
+encoder works through admitted requests on its own GPU and never stalls the
+batch. Measured on the Strix Halo with the encoder on the R9700: two text
+streams decoding while three one-image requests and one eight-image request
+arrive pause at most 0.67 s between tokens; four concurrent image answers of
+256 tokens finish in 37.5 s (27 tok/s in total).
+
+The staging caches, one per slot, are allocated at startup and logged
+(`staging caches 286 MB` for 4 slots at `--max-ctx 8192`). Image requests
+beyond the free slots wait in the queue. Under KV pressure the scheduler never
+parks an image request for recompute (token ids cannot rebuild image rows); it
+suspends or parks a text request instead. `/props` reports the effective
+capability in `capabilities.image_input_supported` after backend
 initialization.
 
 ## Qwen3.5 / Qwen3.8

server/src/common/concurrency/seq_engine.h

@@ -333,6 +333,10 @@ class SeqEngine {
         error = "engine does not support KV eviction";
         return false;
     }
+    // False when the slot's KV cannot be rebuilt from its token history (for
+    // example image rows that prefill from pixels): the scheduler then picks
+    // another eviction victim.
+    virtual bool kv_recomputable(int) const { return true; }
 
     // True when a parked slot's resume reservation fits current free pool
     // capacity with headroom for the resident cohort's next step — the

server/src/deepseek4/deepseek4_backend.h

@@ -25,7 +25,11 @@
 #include "ggml.h"
 #include "ggml-backend.h"
 
+#include <atomic>
+#include <condition_variable>
+#include <deque>
 #include <memory>
+#include <mutex>
 #include <random>
 #include <string>
 #include <thread>
@@ -135,8 +139,18 @@ class DeepSeek4Backend : public ModelBackend {
     // Owned backend for the vision encoder when --mmproj-device names a GPU
     // other than the target's; null when the encoder shares backend_.
     ggml_backend_t         vision_backend_ = nullptr;
-    // Encodes images on vision_backend_ while prefill consumes them.
-    std::thread            image_stream_;
+    // Encoder worker on vision_backend_: encodes queued image requests in
+    // order and publishes each image as it lands, so neither the scheduler
+    // nor prefill waits for a whole request. Started on first use.
+    std::thread            encode_worker_;
+    std::mutex             encode_mutex_;
+    std::condition_variable encode_ready_;
+    std::deque<std::shared_ptr<const DeepSeek4ImagePrompt>> encode_queue_;
+    std::atomic<bool>      encode_stop_{false};  // also read by the worker's cancel check
+    vision::ImageSentinels image_sentinels_;
+    // Batched image serving: one single-request staging cache per slot
+    // (slot 0 uses cache_), allocated at startup.
+    std::vector<std::unique_ptr<DeepSeek4Cache>> image_staging_caches_;
     vision::ImageAdmissionReserves image_reserves_;
 
     // Sampler
@@ -203,24 +217,35 @@ class DeepSeek4Backend : public ModelBackend {
                    int prefix_tokens = 0);
     bool load_vision();
     bool init_single_gpu_vision();
-    // Waits for a streaming image encode started by materialize_images.
-    void join_image_stream();
-    // Batched serving. encode_image_request materializes an image request's
-    // rows; prefill_staged fills each request's first `prefix` tokens into
-    // its own staging cache in shared layer-major passes (expert weights read
-    // once per pass for every request in it).
-    struct StagedPrefill {
-        ImagePromptHandle images;
-        const std::vector<int32_t> * prompt = nullptr;
-        int prefix = 0;
-        DeepSeek4Cache * staging = nullptr;
-        bool ok = false;
-        std::string error;
-    };
+    // Encodes one image with the vision runtime (caller serialises use).
+    bool encode_one_image(const vision::PromptImage & image, vision::ImageRaster & raster,
+                          std::string & error);
+    // Queues an image request on the encoder worker (--mmproj-device only).
+    void enqueue_image_encode(std::shared_ptr<const DeepSeek4ImagePrompt> images);
+    void encode_worker_loop();
+    void cancel_image_encode(const ImagePromptPayload & images) const;
+    // Stops the encoder worker (failing queued requests), then frees vision_.
+    void release_vision();
+    // Batched serving. A staged prefill fills one request's first `prefix`
+    // tokens into its slot's staging cache over several steps, in shared
+    // layer-major passes (expert weights read once per pass for all of them)
+    // that the engine advances a few layers per step.
+    using StagedPrefill = DeepSeek4StagedPrefill;
+    DeepSeek4Cache * image_staging_cache(int slot);
+    // Starts encoding an admitted image request: queued on the encoder
+    // worker with --mmproj-device, otherwise encoded here.
     bool encode_image_request(const std::vector<int32_t> & prompt, const ImagePromptHandle & images,
                               std::string & error);
-    void prefill_staged(std::vector<StagedPrefill> & batch);
-    bool materialize_images(const DeepSeek4ImagePrompt & images,
+    bool begin_staged_prefill(StagedPrefill & item);
+    // Starts one shared pass over the ready, unfinished items, about
+    // `row_budget` rows in total (a whole image block may exceed it); `rows`
+    // gets each item's share. False when no item is ready (or all failed).
+    bool begin_staged_pass(const std::vector<StagedPrefill *> & items, int row_budget,
+                           DeepSeek4PrefillPass & pass, std::vector<int> & rows);
+    // Waits up to `timeout_ms` for the next rows of an item to have their
+    // images encoded, so an otherwise idle scheduler does not spin.
+    void wait_staged_ready(const StagedPrefill & item, int row_budget, int timeout_ms) const;
+    bool materialize_images(const std::shared_ptr<const DeepSeek4ImagePrompt> & images,
                             const DaemonIO & io, std::string & error);
 
     // Generate after either a fresh prefill or a restored prefix. kv_offset is