diff --git a/Dockerfile.rocm b/Dockerfile.rocm index d16dc91d3..4ea7a1d74 100644 --- a/Dockerfile.rocm +++ b/Dockerfile.rocm @@ -46,7 +46,8 @@ ENV PATH=/opt/rocm/bin:/opt/rocm/lib/llvm/bin:${PATH} # hipCUB/rocPRIM: ggml's ROCm argsort/top-k kernels use their headers for the # large-N device-sort path. rocWMMA is also required by the DS4 indexed # prefill kernel. The ROCm apt repository ships these headers separately from -# the base HIP toolchain. +# the base HIP toolchain. hipBLASLt is optional for ggml-hip: with its -dev +# package present the DS4V vision ops are built, without it --mmproj is refused. # (deps/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt does find_package(hipblas) # for its BLAS matmul path). The rocm/dev-ubuntu base ships the HIP toolchain # but NOT the math libs, so they are installed explicitly from the ROCm apt @@ -59,6 +60,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ git-lfs \ hipblas-dev \ + hipblaslt-dev \ hipcub-dev \ libcurl4-openssl-dev \ ninja-build \ @@ -151,6 +153,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ hipblas \ + hipblaslt \ libgomp1 \ pciutils \ rocblas \ diff --git a/README.md b/README.md index 095385012..e99f4682a 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,7 @@ See [Continuous batching in Lucebox](https://www.lucebox.com/blog/continuous-bat | OpenAI Chat Completions, Responses, and Anthropic Messages | [API reference](server/docs/API.md) | | CUDA, HIP, and mixed-device placement | [Mixed-backend guide](server/docs/MIXED_BACKEND.md) | | DeepSeek V4 single-device and heterogeneous profiles | [DeepSeek V4 guide](server/docs/DS4.md) | +| Image input (Qwen3.8, DeepSeek V4 Flash Vision) | [Image input guide](docs/image-input.md) | | Environment variables | [Environment reference](server/docs/ENVIRONMENT.md) | | Server internals | [Architecture](server/docs/ARCHITECTURE.md) | | Client integration and qualification | [Harness guide](harness/README.md) | diff --git a/docs/ds4v-mmproj.md b/docs/ds4v-mmproj.md new file mode 100644 index 000000000..6d989de55 --- /dev/null +++ b/docs/ds4v-mmproj.md @@ -0,0 +1,37 @@ +# DeepSeek-V4 vision projector export + +`server/tools/export_ds4v_mmproj.py` extracts the vision tower and aligner from +the verified DeepSeek-V4 parent model without loading or changing tensor values. +It requires only Python's standard library. The output is published as +[`DeepSeek-V4-Flash-Vision-Exp-mmproj-BF16.gguf`](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3); +run the exporter only to rebuild it. + +```bash +python3 server/tools/export_ds4v_mmproj.py \ + /models/DeepSeek-V4-Flash-Vision-Uncensored \ + /models/ds4v-mmproj.gguf +``` + +The output path must not exist. The exporter validates `config.json`, the +safetensors index, all 267 required names and shapes, BF16 payload lengths, +source bounds, and shard paths before it writes. It publishes the finished file +atomically and removes its temporary file after any pre-publication failure. + +The GGUF keeps every `vision.*`, `aligner.*`, and `image_*` name, shape, dtype, +and payload byte unchanged. `general.architecture` is `deepseek4_vision`. +Required `deepseek4.vision.*` metadata records the 32-block, width-1024, +16-head tower; head width 64; patch size 14; intermediate width 2816; 2D RoPE +layout and theta 10000; ratio-3, width-9216 aligner; language width 4096; +vocabulary 129280; RMS epsilon `1e-6`; image token/pixel/aspect bounds; RGB +mean/std `0.5`; channel-major patches; N-layout recipe version 1 with +compression alignment 4; bottom/right aligner padding; channel-first unfold; +and exact GELU. + +Run the focused synthetic suite with: + +```bash +python3 -m unittest -v server.tests.test_export_ds4v_mmproj +``` + +When NumPy is available, the suite also opens the result with llama.cpp's +vendored `GGUFReader`, independently of the exporter's writer. diff --git a/docs/image-input.md b/docs/image-input.md new file mode 100644 index 000000000..d4a264b9d --- /dev/null +++ b/docs/image-input.md @@ -0,0 +1,300 @@ +# Image input + +The server accepts JPEG and PNG images through OpenAI chat completions when a +model is started with its vision projector, `--mmproj `. Without +`--mmproj` nothing in the text serving path changes. + +| Model | Decoder | Projector | Runs on | +| --- | --- | --- | --- | +| Qwen3.8-27B | [`Qwen3.8-27B-IQ4_XS-pure.gguf`](https://huggingface.co/Lucebox/Qwen3.8-27B-IQ4_XS-fast-GGUF) (any Qwen3.5 / Qwen3.8 dense GGUF works) | [`Qwen3.8-27B-mmproj-Q8_0.gguf`](https://huggingface.co/Lucebox/Qwen3.8-27B-IQ4_XS-fast-GGUF), or any published `qwen3vl_merger` mmproj | one GPU, any backend | +| DeepSeek V4 Flash Vision (DS4V) | [`DeepSeek-V4-Flash-Vision-Exp-ROCMFPX-MIX-STRIX.gguf`](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3) | [`DeepSeek-V4-Flash-Vision-Exp-mmproj-BF16.gguf`](https://huggingface.co/Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3) | HIP: a Strix Halo alone, or R9700 + Strix Halo | + +**Status: experimental.** Both models answer image questions correctly end to +end; see each model's verification notes for what has and has not been +measured. + +## Quick start + +Build the server as in the [README](../README.md#run-the-server). DS4V also +needs hipBLASLt at build time (`hipblaslt-dev` on ROCm; CMake prints +`hipBLASLt found: building the DS4V vision ops`). + +### Qwen3.8-27B on one GPU (R9700) + +```bash +hf download Lucebox/Qwen3.8-27B-IQ4_XS-fast-GGUF \ + Qwen3.8-27B-IQ4_XS-pure.gguf Qwen3.8-27B-mmproj-Q8_0.gguf --local-dir models +hf download Lucebox/Qwen3.8-27B-DFlash2-GGUF \ + Qwen3.8-27B-DFlash2-Q8_0.gguf --local-dir models + +./server/build-hip/luce_server models/Qwen3.8-27B-IQ4_XS-pure.gguf \ + --target-device hip:0 \ + --draft models/Qwen3.8-27B-DFlash2-Q8_0.gguf --draft-device hip:0 \ + --draft-block-size 16 --max-ctx 32768 \ + --cache-type-k q8_0 --cache-type-v q8_0 \ + --mmproj models/Qwen3.8-27B-mmproj-Q8_0.gguf \ + --port 8216 +``` + +About 21 GiB of VRAM at the peak of an image request. Text requests keep the +DFlash2 drafter; image requests decode without it. + +### DeepSeek V4 Flash Vision on a Strix Halo + +```bash +hf download Lucebox/DeepSeek-V4-Flash-0731-ROCmFP3 \ + DeepSeek-V4-Flash-Vision-Exp-ROCMFPX-MIX-STRIX.gguf \ + DeepSeek-V4-Flash-Vision-Exp-mmproj-BF16.gguf --local-dir models +hf download Lucebox/DeepSeek-V4-Flash-0731-DSpark-GGUF \ + DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf --local-dir models + +LUCE_DS4_SPEC=1 \ +LUCE_DS4_DRAFT=models/DeepSeek-V4-Flash-0731-DSpark-draft-Q4RMFP4-denseF16.gguf \ +LUCE_DS4_SPARSE_DECODE_FLASH=1 \ +./server/build-hip/luce_server models/DeepSeek-V4-Flash-Vision-Exp-ROCMFPX-MIX-STRIX.gguf \ + --target-device hip:0 --max-ctx 131072 --chunk 8192 \ + --cache-type-k q4_0 --cache-type-v q4_0 \ + --ds4-fused-decode --ds4-fused-verify-f16-kv \ + --ds4-expert-top-k 6 --ds4-prefill sparse \ + --mmproj models/DeepSeek-V4-Flash-Vision-Exp-mmproj-BF16.gguf \ + --port 8216 +``` + +`hip:0` must be the Strix Halo; on a host with a discrete GPU too, expose the +Strix Halo alone with `HIP_VISIBLE_DEVICES`. This is the text model's published +launch plus `--mmproj`: the Vision file replaces +`DeepSeek-V4-Flash-0731-ROCMFPX-MIX-STRIX.gguf` for text as well and decodes +at least as fast (numbers below). For R9700 + Strix Halo see [DS4V](#ds4v) below. + +### Send an image + +```bash +IMG=$(base64 < photo.png | tr -d '\n') +curl -s http://127.0.0.1:8216/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"messages":[{"role":"user","content":[ + {"type":"text","text":"What does this chart show?"}, + {"type":"image_url","image_url":{"url":"data:image/png;base64,'"$IMG"'"}}]}], + "max_tokens":256}' +``` + +`GET /props` reports `capabilities.image_input_supported: true` once the +projector has loaded. In the Docker images, set `LUCE_MMPROJ` to the projector +path inside the container. + +## Request contract + +Use `POST /v1/chat/completions` with user-message content parts in display order: + +```json +{ + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image."}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}} + ] + }], + "max_tokens": 128 +} +``` + +Only base64 JPEG/PNG data URLs are supported. Remote URLs, images outside user +content arrays, and image parts through other API formats are rejected. A +request carries at most four images, 16 MiB encoded each and 32 MiB combined. +Decoder pixel and aspect limits also apply. A model's image marker cannot be supplied +as ordinary text. + +The server expands image markers after final rendering and tokenization, and +the expanded image tokens count toward context and usage. Image requests use +plain autoregressive decoding and bypass the token-keyed prefix, disk and +agent-turn caches and prompt compression: tokens alone do not identify an +image. Text requests on the same server keep speculative decoding and caching. + +Layer or tensor splitting across GPUs, remote target shards, concurrent +sequence scheduling (`--max-concurrency`) and upstream forwarding do not +support images. `/props` reports the effective capability in +`capabilities.image_input_supported` after backend initialization. + +## Qwen3.5 / Qwen3.8 + +The launch is in the [quick start](#qwen38-27b-on-one-gpu-r9700). + +The projector is read directly from the published `clip` file, BF16, F16 or +Q8_0; Q8_0 is recommended (below). Projectors with +deepstack branches (Qwen3-VL) are refused. An image is resized the way the +model was trained (bicubic, both sides to a multiple of 32 pixels) and costs +one token per 32x32 pixels, between 64 and 1,024 tokens; larger images are +scaled down to the cap. + +Image tokens take two-dimensional rotary positions, so positions run behind +token counts after an image. Prefill handles that in its normal chunk loop; +decoding carries the offset for the rest of the request. + +### Verification status + +Covered by `test_qwen35_image`: target sizes against the model's reference +resize rule, the tower's patch order and position table sampling, marker +expansion, rotary positions, and image rows that straddle prefill chunks. + +Measured on an R9700 alone with the DFlash2 drafter, thinking off. With the +Lucebox `Qwen3.8-27B-IQ4_XS-pure` file and a Q8_0 projector: + +- 220 seeded questions from `lmms-lab/ai2d` and `lmms-lab/ChartQA` with + lmms-eval prompts: AI2D 90/100, ChartQA relaxed accuracy 56/60 (augmented) + and 42/60 (human). Image prompts prefill in 0.56 s on average; one to four + images per request all answer correctly (four images, 2,495 tokens: 3.2 s). +- Text decodes at 56 to 117 tok/s on 256-token answers (84 on average); image + requests decode without the drafter at about 36 tok/s. + +With unsloth's UD-IQ4_XS file and the published BF16 projector: + +- 220 seeded questions from `lmms-lab/ai2d` and `lmms-lab/ChartQA` with + lmms-eval prompts: AI2D 85/100, ChartQA relaxed accuracy 55/60 (augmented) + and 43/60 (human), no errors. Image prompts average 448 tokens and prefill in + 0.71 s (largest 1,068 tokens, 1.8 s); decode runs at 31 to 35 tok/s. +- A projector with its weight matrices in Q8_0 (rows that are not a multiple + of 32 stay F16) encodes a 975-token image in 443 ms instead of 677 ms with + the BF16 file, with the same scores on the 220 questions and 216 identical + answers. Prefer one when available. +- llama.cpp (HIP build, `-fa on`, same GGUF, projector and image cap) answers + the same on every test image; on a 1,012-token image prompt it prefills in + 1.65 s to our 1.68 s, on a 323-token one in 0.61 s to our 0.50 s. +- An image inside a 6,271-token prompt, two images in one request, and a + follow-up turn after an image all answer correctly. +- Text requests are byte-identical to a build without image support, at the + same speed, with or without a projector loaded (five prompts up to 19.6K + tokens). The projector adds 0.9 GiB of VRAM; the peak during image requests + was 21.6 GiB against 20.8 GiB for text. +- The same requests answer correctly on a Strix Halo alone, where a + 1,012-token image prompt prefills in 4.6 s and decodes at 14 tok/s. + +Not yet established: a comparison against the reference implementation on the +same questions, and CUDA. The tower uses only standard ggml +operators, so nothing in it is HIP specific. + +## DS4V + +The server must be built with hipBLASLt available (the `hipblaslt-dev` package +on ROCm). CMake reports `hipBLASLt found: building the DS4V vision ops`; a build +without it refuses `--mmproj` for this model at startup. + +Image input needs Linux HIP, a DeepSeek4 decoder whose GGUF carries the image +router biases, `--ds4-prefill sparse`, and `--mmproj` pointing at the published projector (or one [exported with our +tool](ds4v-mmproj.md)). Two layouts work: + +- **One GPU holding the whole model** (for example a Strix Halo): nothing else + to set. The projector is loaded after the weights and must fit beside them. +- **Two GPUs splitting the experts in process** (for example R9700 + Strix + Halo): `LUCE_DS4_MOE_TP=1`, `LUCE_DS4_MOE_TP_INPROC=1`, and + `LUCE_DS4_MOE_TP_GPU` selecting the second device, with `--target-device` + on the first. Device ordinals must match the host's actual topology. + +Remote expert IPC, all-on-secondary placement, experts kept on the CPU and +dense prefill do not support images. + +Published llama.cpp conversions of the decoder load directly (image router +bias named `blk.N.exp_probs_b_vl.bias`, no `deepseek4.vocab_size` key). Split +GGUF files and llama.cpp's `clip` projector files are not read yet. + +A decoder in our own ROCMFP MIX format comes from `tools/ds4_mix_converter` +run on the Vision-Exp checkpoint. It follows the shipped DeepSeek-V4-Flash +recipe: routed gate and up experts in fp2, down experts in fp2 on the shipped +layer set and fp3 elsewhere, dense projections in ROCmFP4, the token embedding +in Q6_K, codebooks embedded in the GGUF (one file, about 100 GB). It keeps the +image router biases. Pass `--imatrix` with an importance matrix (llama.cpp's +per-expert layout is used expert by expert; the community publishes one for +this model) or `--absmax-only`. The converter uses every core: about 40 minutes +for this checkpoint on 32 cores. + +One image request may be outstanding per backend. Its admission lease remains +with the immutable payload through queueing and generation; another image +request is rejected until that payload is released. This bounds simultaneous +preprocessing and prepared-image memory. Text requests retain the normal queue. + +The server expands image markers after final rendering and tokenization. +Expanded image tokens count toward context and usage. Image blocks remain +whole during prefill, image rows use their learned routing bias, and raw +attention is bidirectional within each image's visible span. The projector's +tile permutation is applied once when assembling rows with named sentinel +embeddings. All chunks are capped at 1,024 tokens while a projector is loaded. + +The image payload survives request copies and retry paths. Failed or cancelled +multi-image encoding publishes no partial embedding matrices. + +### Memory + +The projector is validated and loaded before expert placement. Admission counts +actual selected owner tensor sizes, allocation alignment, MIX tables, copy +staging, future KV, and explicit execution reserves. Host and integrated-device +charges share one physical-memory budget. Before image decoding, the server +checks host availability; before encoding, it synchronizes and releases +disposable decoder, owner, and draft graphs and checks live device/host +availability again. KV, saved snapshots, and draft weights remain reflected in +that live measurement. Reservations are conservative policy, not a guarantee +against unrelated concurrent allocations. + +### Verification status + +Covered by unit tests in the main build: image transport and request policy, +prompt expansion and ownership, embedding assembly and cancellation, image +spans and the expert budget, plus the decoder loader and image-batch admission +tests in `test_deepseek4_unit`. + +Measured with the public `DeepSeek-V4-Flash-Vision-Exp` Q2_K_S decoder and +the exported projector, on a Strix Halo alone and on R9700 + Strix Halo, 220 +seeded questions from `lmms-lab/ai2d` and `lmms-lab/ChartQA` with lmms-eval +prompts: AI2D 85/100, ChartQA relaxed accuracy 55/60 (augmented) and 43/60 +(human). Both layouts score the same and give word-identical answers on 213 of +220 questions. An image request prefills in about 4 s and decodes at about +23 tok/s. + +With our own ROCMFP MIX conversion of the same checkpoint (per-expert +importance matrix, the shipped recipe above), on a Strix Halo alone at top-k 6: + +- Against the MXFP4 reference (native FP4 experts) on 8,176 wikitext-2 tokens: + KL 0.464 mean, 0.102 median, top-1 agreement 78.4%, perplexity 4.14 against + 2.82. The community Q2_K_S scores KL 0.511 in our engine (0.523 in + llama.cpp) and perplexity 4.22. +- AI2D 84/100, ChartQA 54/60 and 40/60 (the Q2_K_S: 85, 55, 43); the sanity + and one-to-four-image sets are all correct. +- With the published DSpark drafter and fused decode and verify, text decodes + at 25 to 37 tok/s on 256-token answers (30 mean), as fast as the shipped + text model; image requests decode without the drafter at about 22 tok/s. + +Not yet established: + +- The vision tower misses the fixed 0.9995 feature-cosine gate against the + reference implementation: 0.99906 on the Radeon RX 7900 XT it was developed + on, 0.99823 on CPU. Embeddings pass; features do not. +- No comparison against the reference implementation on the same questions. + +## Code layout + +Shared by every model: + +| Piece | Where | +| --- | --- | +| Reading images out of a request, limits, redaction | `server/src/server/image_input.*` | +| JPEG and PNG decoding to RGB | `server/src/common/vision/image_decode.*`, codecs in `server/cmake/ImageCodecs.cmake` | +| Bicubic resizing that matches Pillow byte for byte | `server/src/common/vision/image_resize.*` | +| Reading a published `clip`-format projector file | `server/src/common/vision/mmproj_file.*` | +| Image positions in a prompt, batches that keep an image whole | `server/src/common/vision/image_spans.h` | +| The backend contract | `supports_images`, `image_placeholder`, `prepare_images` in `server/src/common/model_backend.h`; `GenerateRequest::images` in `server/src/common/generation_types.h` | + +DS4V only, all under `server/src/deepseek4/`: resizing and patching +(`deepseek4_vision_preprocess`), the vision tower (`deepseek4_vision`), marker +expansion and embedding assembly (`deepseek4_image_prompt`, +`deepseek4_image_assembly`), attention visibility and expert routing for image +rows (`deepseek4_image_policy`), and memory admission +(`deepseek4_image_admission`). + +Qwen3.5 / Qwen3.8 only, all under `server/src/qwen35/`: the vision tower and +its preprocessing (`qwen35_vision`), marker expansion and rotary positions +(`qwen35_image_prompt`), what a request carries (`qwen35_image_request.h`), and +the backend's three contract methods (`qwen35_backend_images.cpp`). Prefill and +decode changes are a few lines in `qwen35_backend.cpp`. + +Another model needs its own preprocessing, tower and prompt expansion, and its +backend implements the three contract methods. Nothing in the HTTP server or in +`common/vision` names a model. diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index badc06b3a..8d32b7fb7 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -381,6 +381,25 @@ if(LUCE_MIXED_GGML_SHARED) set(BUILD_SHARED_LIBS ON) endif() add_subdirectory(deps/llama.cpp/ggml EXCLUDE_FROM_ALL) + +option(LUCE_DS4_MIX_CONVERTER + "Build the CPU-only DeepSeek-V4 safetensors to MIX GGUF converter" ON) +# The Docker build contexts do not copy server/tools. +if(LUCE_DS4_MIX_CONVERTER AND + EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/ds4_mix_converter/ds4_mix_converter.cpp") + add_executable(ds4_mix_converter + tools/ds4_mix_converter/ds4_mix_converter.cpp) + target_include_directories(ds4_mix_converter PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include + ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/rocmfpx + ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/rocmfp4) + target_link_libraries(ds4_mix_converter PRIVATE + ggml-base nlohmann_json::nlohmann_json) + if(UNIX) + target_link_libraries(ds4_mix_converter PRIVATE m) + endif() +endif() + if(LUCE_MIXED_GGML_SHARED) if(_luce_build_shared_libs_was_defined) set(BUILD_SHARED_LIBS "${_luce_saved_build_shared_libs}") @@ -499,6 +518,17 @@ add_library(luce_common STATIC src/deepseek4/deepseek4_target_shard_ipc_daemon.cpp src/deepseek4/deepseek4_dspark.cpp src/deepseek4/deepseek4_dspark_spec.cpp + src/deepseek4/deepseek4_vision.cpp + src/common/gpu_page_pool.cpp + src/common/vision/image_decode.cpp + src/common/vision/image_resize.cpp + src/common/vision/mmproj_file.cpp + src/deepseek4/deepseek4_vision_preprocess.cpp + src/deepseek4/deepseek4_image_prompt.cpp + src/deepseek4/deepseek4_image_assembly.cpp + src/deepseek4/deepseek4_image_admission.cpp + src/deepseek4/deepseek4_image_policy.cpp + src/server/image_input.cpp src/flashprefill_q8.cpp src/kv_cache.cpp src/kv_quant.cpp @@ -555,6 +585,9 @@ add_library(luce_common STATIC src/qwen35/qwen35_target_shard_ipc_daemon.cpp src/qwen35/layer_split_daemon.cpp src/qwen35/qwen35_backend.cpp + src/qwen35/qwen35_backend_images.cpp + src/qwen35/qwen35_image_prompt.cpp + src/qwen35/qwen35_vision.cpp src/qwen35/qwen35_tensor_parallel.cpp src/qwen35/concurrency/qwen35_seq_engine.cpp src/qwen35/qwen35_layer_split_adapter.cpp @@ -873,6 +906,25 @@ if(LUCE_ENABLE_BSA) endif() endif() +# JPEG and PNG decoders for image input (common/vision/image_decode). They are +# downloaded at build time; an offline or text-only build can turn them off and +# image requests are then refused. +option(LUCE_IMAGE_CODECS "Download and build the JPEG/PNG decoders for image input" ON) +if(LUCE_IMAGE_CODECS) + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ImageCodecs.cmake") + install(FILES cmake/ImageCodecs.NOTICES.md + DESTINATION share/licenses/image-codecs + RENAME THIRD_PARTY_NOTICES.md) + install(FILES "${IMAGE_CODEC_JPEG_SOURCE_DIR}/LICENSE.md" + "${IMAGE_CODEC_JPEG_SOURCE_DIR}/README.ijg" + DESTINATION share/licenses/image-codecs/libjpeg-turbo + OPTIONAL) + set(_luce_image_codec_libs image_codec_jpeg image_codec_png) +else() + set(_luce_image_codec_libs) + target_compile_definitions(luce_common PRIVATE LUCE_NO_IMAGE_CODECS) +endif() + target_link_libraries(luce_common PUBLIC ggml @@ -880,6 +932,7 @@ target_link_libraries(luce_common ggml-base nlohmann_json::nlohmann_json PRIVATE + ${_luce_image_codec_libs} ${CMAKE_DL_LIBS} ) # OpenMP for parallel MoE expert compute kernel (saturate memory bandwidth). @@ -1010,6 +1063,16 @@ if(LUCE_TESTS) endif() list(APPEND _raw_unit_test_targets test_rocmfpx) + add_executable(test_ds4_mix_converter test/test_ds4v_mix_converter.cpp test/test_unit_main.cpp) + target_include_directories(test_ds4_mix_converter PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/include + ${CMAKE_CURRENT_SOURCE_DIR}/deps/llama.cpp/ggml/rocmfpx) + target_link_libraries(test_ds4_mix_converter PRIVATE ggml-base) + if(UNIX) + target_link_libraries(test_ds4_mix_converter PRIVATE m) + endif() + list(APPEND _raw_unit_test_targets test_ds4_mix_converter) + if(LUCE_GPU_BACKEND STREQUAL "hip") add_executable(test_rocmfp4_hip_tail test/test_rocmfp4_hip_tail.cpp) set_source_files_properties(test/test_rocmfp4_hip_tail.cpp PROPERTIES LANGUAGE HIP) @@ -1840,6 +1903,46 @@ if(LUCE_TESTS) endif() # ─── Unit tests (no GPU, no model files) ──────────────────────────── + # Shared image input: request transport and JPEG/PNG decoding. + add_executable(test_image_input test/test_image_input.cpp src/server/image_input.cpp) + target_include_directories(test_image_input PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + target_link_libraries(test_image_input PRIVATE nlohmann_json::nlohmann_json) + if(LUCE_IMAGE_CODECS) + add_executable(test_image_decode test/test_image_decode.cpp src/common/vision/image_decode.cpp) + target_include_directories(test_image_decode PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + target_link_libraries(test_image_decode PRIVATE image_codec_jpeg image_codec_png) + list(APPEND _raw_unit_test_targets test_image_decode) + endif() + add_executable(test_image_resize test/test_image_resize.cpp src/common/vision/image_resize.cpp) + target_include_directories(test_image_resize PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + add_executable(test_gpu_page_pool test/test_gpu_page_pool.cpp src/common/gpu_page_pool.cpp) + target_include_directories(test_gpu_page_pool PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + add_executable(test_qwen35_image test/test_qwen35_image.cpp + src/qwen35/qwen35_image_prompt.cpp src/qwen35/qwen35_vision.cpp + src/common/vision/mmproj_file.cpp src/common/vision/image_resize.cpp) + target_include_directories(test_qwen35_image PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + target_link_libraries(test_qwen35_image PRIVATE ggml ggml-base) + list(APPEND _raw_unit_test_targets test_image_input test_image_resize test_gpu_page_pool + test_qwen35_image) + + # DS4V image units: each test builds only the unit it covers. + foreach(_ds4v_unit assembly integration policy prompt) + add_executable(test_ds4v_image_${_ds4v_unit} test/test_ds4v_image_${_ds4v_unit}.cpp) + target_include_directories(test_ds4v_image_${_ds4v_unit} PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/src/deepseek4) + list(APPEND _raw_unit_test_targets test_ds4v_image_${_ds4v_unit}) + endforeach() + target_sources(test_ds4v_image_assembly PRIVATE src/deepseek4/deepseek4_image_assembly.cpp) + target_sources(test_ds4v_image_policy PRIVATE src/deepseek4/deepseek4_image_policy.cpp) + target_sources(test_ds4v_image_prompt PRIVATE + src/deepseek4/deepseek4_image_prompt.cpp + src/deepseek4/deepseek4_vision_preprocess.cpp + src/common/vision/image_resize.cpp) + + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_moe_source_page_range.cpp") + add_executable(test_moe_source_page_range test/test_moe_source_page_range.cpp) + list(APPEND _raw_unit_test_targets test_moe_source_page_range) + endif() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/test/test_moe_input_ready.cpp") add_executable(test_moe_input_ready test/test_moe_input_ready.cpp) target_include_directories(test_moe_input_ready PRIVATE diff --git a/server/cmake/ImageCodecs.NOTICES.md b/server/cmake/ImageCodecs.NOTICES.md new file mode 100644 index 000000000..33bf7ed2d --- /dev/null +++ b/server/cmake/ImageCodecs.NOTICES.md @@ -0,0 +1,69 @@ +# Third-party notices + +## Pillow 12.3.0 + +The resize implementation follows Pillow 12.3.0 `src/libImaging/Resample.c`. + +The Python Imaging Library (PIL) is + + Copyright © 1997-2011 by Secret Labs AB + Copyright © 1995-2011 by Fredrik Lundh and contributors + +Pillow is the friendly PIL fork. It is + + Copyright © 2010 by Jeffrey 'Alex' Clark and contributors + +Like PIL, Pillow is licensed under the open source MIT-CMU License: + +By obtaining, using, and/or copying this software and/or its associated +documentation, you agree that you have read, understood, and will comply +with the following terms and conditions: + +Permission to use, copy, modify and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appears in all copies, and that +both that copyright notice and this permission notice appear in supporting +documentation, and that the name of Secret Labs AB or the author not be +used in advertising or publicity pertaining to distribution of the software +without specific, written prior permission. + +SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, +INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +## LodePNG + +LodePNG version 20260119 + +Copyright (c) 2005-2026 Lode Vandevenne + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. + +## libjpeg-turbo 3.1.4.1 + +The probe statically links the libjpeg API library and disables the TurboJPEG API library. + +This software is based in part on the work of the Independent JPEG Group. + +The downloaded source archive retains its complete `LICENSE.md` and `README.ijg` notices. diff --git a/server/cmake/ImageCodecs.cmake b/server/cmake/ImageCodecs.cmake new file mode 100644 index 000000000..c44aa143b --- /dev/null +++ b/server/cmake/ImageCodecs.cmake @@ -0,0 +1,84 @@ +# JPEG and PNG decoders behind common/vision/image_decode: libjpeg-turbo from +# its pinned release archive, lodepng from two files at a pinned commit. License +# texts are in ImageCodecs.NOTICES.md and the libjpeg-turbo archive. +include_guard(GLOBAL) + +include(ExternalProject) + +# DOWNLOAD_EXTRACT_TIMESTAMP exists from CMake 3.24; older releases would read +# it as part of URL_HASH. +set(IMAGE_CODEC_EXTRACT_TIMESTAMP) +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24) + set(IMAGE_CODEC_EXTRACT_TIMESTAMP DOWNLOAD_EXTRACT_TIMESTAMP TRUE) +endif() + +set(IMAGE_CODEC_JPEG_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libjpeg-turbo-prefix) +set(IMAGE_CODEC_JPEG_ARCHIVE_NAME jpeg) +if(MSVC OR CMAKE_C_SIMULATE_ID STREQUAL "MSVC") + set(IMAGE_CODEC_JPEG_ARCHIVE_NAME jpeg-static) +endif() +set(IMAGE_CODEC_JPEG_ARCHIVE + ${IMAGE_CODEC_JPEG_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${IMAGE_CODEC_JPEG_ARCHIVE_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}) +file(MAKE_DIRECTORY ${IMAGE_CODEC_JPEG_PREFIX}/include) +ExternalProject_Add(libjpeg_turbo_external + URL https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.1.4.1/libjpeg-turbo-3.1.4.1.tar.gz + URL_HASH SHA256=ecae8008e2cc9ade2f2c1bb9d5e6d4fb73e7c433866a056bd82980741571a022 + ${IMAGE_CODEC_EXTRACT_TIMESTAMP} + CMAKE_ARGS + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_PREFIX=${IMAGE_CODEC_JPEG_PREFIX} + -DCMAKE_INSTALL_LIBDIR=lib + -DENABLE_SHARED=OFF + -DENABLE_STATIC=ON + -DWITH_TOOLS=OFF + -DWITH_TESTS=OFF + -DWITH_SIMD=OFF + -DWITH_TURBOJPEG=OFF + BUILD_COMMAND ${CMAKE_COMMAND} --build --parallel 2 + BUILD_BYPRODUCTS ${IMAGE_CODEC_JPEG_ARCHIVE}) +add_library(image_codec_jpeg STATIC IMPORTED GLOBAL) +set_target_properties(image_codec_jpeg PROPERTIES + IMPORTED_LOCATION ${IMAGE_CODEC_JPEG_ARCHIVE} + INTERFACE_INCLUDE_DIRECTORIES ${IMAGE_CODEC_JPEG_PREFIX}/include) +add_dependencies(image_codec_jpeg libjpeg_turbo_external) + +# lodepng has no release archives and GitHub's commit archives are not +# byte-stable, so fetch its two files at a pinned commit (raw files are) and +# check each against its hash. Retried because CI runners drop downloads. +set(IMAGE_CODEC_PNG_COMMIT ed6fe5825c6a4fbb7f58ab35a4231c7543cd452a) +set(IMAGE_CODEC_PNG_DIR ${CMAKE_CURRENT_BINARY_DIR}/lodepng-${IMAGE_CODEC_PNG_COMMIT}) +foreach(entry + "lodepng.cpp=d98e1f40d303c1038a096ebf93b413a565a91cf2c72b9d2fa5c625c4279c3cb6" + "lodepng.h=23c27abb06883ed98184d16d0b20771b526dca1e8e13236c2397316769c0dc8b") + string(REPLACE "=" ";" entry "${entry}") + list(GET entry 0 file) + list(GET entry 1 hash) + set(target ${IMAGE_CODEC_PNG_DIR}/${file}) + set(have "") + foreach(attempt RANGE 1 3) + if(EXISTS ${target}) + file(SHA256 ${target} have) + if(have STREQUAL hash) + break() + endif() + file(REMOVE ${target}) + endif() + file(DOWNLOAD + https://raw.githubusercontent.com/lvandeve/lodepng/${IMAGE_CODEC_PNG_COMMIT}/${file} + ${target} TLS_VERIFY ON STATUS status) + endforeach() + if(EXISTS ${target}) + file(SHA256 ${target} have) + endif() + if(NOT have STREQUAL hash) + message(FATAL_ERROR "lodepng: could not fetch ${file} with SHA256 ${hash} (${status})") + endif() +endforeach() +add_library(image_codec_png STATIC ${IMAGE_CODEC_PNG_DIR}/lodepng.cpp) +target_include_directories(image_codec_png PUBLIC ${IMAGE_CODEC_PNG_DIR}) + +# Expose the original JPEG notices for production installation after the +# dependency build has downloaded its hash-verified source archive. +ExternalProject_Get_Property(libjpeg_turbo_external SOURCE_DIR) +set(IMAGE_CODEC_JPEG_SOURCE_DIR "${SOURCE_DIR}") +unset(SOURCE_DIR) diff --git a/server/deps/llama.cpp/VENDOR.md b/server/deps/llama.cpp/VENDOR.md index ad185a2d9..d9488270d 100644 --- a/server/deps/llama.cpp/VENDOR.md +++ b/server/deps/llama.cpp/VENDOR.md @@ -27,3 +27,20 @@ the snapshot above for AMD heterogeneous MoE execution: These changes are limited to `ggml/`. Keep their public declarations in `ggml/include`, avoid DeepSeek-specific policy in generic kernels, and update this provenance when the patch set is moved to `lucebox-ggml`. + +## Hub-local DS4V HIP vision ops + +Four inference-only ops are appended after the existing ones, so every earlier +op keeps its numeric value: `GGML_OP_MUL_MAT_BIAS_BF16`, +`GGML_OP_RMS_NORM_VISION_F32`, `GGML_OP_SOFT_MAX_VISION_F32` and +`GGML_OP_MUL_MAT_VISION_AV_F32`. Only the DS4V vision tower builds them. They +exist on the HIP backend alone; CPU, CUDA and RPC reject them in `supports_op`, +and graphs that contain them are not captured. + +They are compiled only when CMake finds hipBLASLt (`GGML_HIP_DS4V_VISION`). +Without it the HIP backend builds as before and the server refuses `--mmproj`. +The fused BF16 bias matmul keeps one hipBLASLt handle and one 76 MiB workspace +per backend context that uses it, released when the context is destroyed. + +Rebuild ggml-base, the backends and their consumers together; do not mix older +shared libraries with this header. diff --git a/server/deps/llama.cpp/ggml/include/ggml-cuda.h b/server/deps/llama.cpp/ggml/include/ggml-cuda.h index 0b24bf2c7..55e0a933e 100644 --- a/server/deps/llama.cpp/ggml/include/ggml-cuda.h +++ b/server/deps/llama.cpp/ggml/include/ggml-cuda.h @@ -25,6 +25,14 @@ extern "C" { #define GGML_CUDA_DS4_MIX_MMV_MAX_TOKENS 5 #define GGML_CUDA_DS4_MIX_MMV_PAGED_MAX_TOKENS 16 +// HIP registry-only opt-in DS4V BF16 linear capability (not NVIDIA/CUDA). +// Lookup "ggml_backend_hip_vision_bias_bf16_workspace" as size_t (*)(ggml_backend_t): +// nonzero means the explicit op is available, and returns its retained external +// workspace reservation (76 MiB). Unsupported op shapes must fail, not fallback. +// "ggml_backend_hip_vision_bias_bf16_launches" has the same signature and returns +// actual successful Lt submissions, with or without bias. Registry names retain +// their original spelling; both modes require a matching GGML/HIP library set. + // backend API GGML_BACKEND_API ggml_backend_t ggml_backend_cuda_init(int device); diff --git a/server/deps/llama.cpp/ggml/include/ggml-rpc.h b/server/deps/llama.cpp/ggml/include/ggml-rpc.h index 21fdf726b..2ce9013a2 100644 --- a/server/deps/llama.cpp/ggml/include/ggml-rpc.h +++ b/server/deps/llama.cpp/ggml/include/ggml-rpc.h @@ -8,10 +8,10 @@ extern "C" { #define RPC_PROTO_MAJOR_VERSION 3 #define RPC_PROTO_MINOR_VERSION 6 -#define RPC_PROTO_PATCH_VERSION 5 +#define RPC_PROTO_PATCH_VERSION 8 #ifdef __cplusplus -static_assert(GGML_OP_COUNT == 105, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); +static_assert(GGML_OP_COUNT == 110, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); #endif #define GGML_RPC_MAX_SERVERS 16 diff --git a/server/deps/llama.cpp/ggml/include/ggml.h b/server/deps/llama.cpp/ggml/include/ggml.h index 8ccc7435e..3132fbab6 100644 --- a/server/deps/llama.cpp/ggml/include/ggml.h +++ b/server/deps/llama.cpp/ggml/include/ggml.h @@ -621,6 +621,10 @@ extern "C" { GGML_OP_PAGED_ATTN, + GGML_OP_MUL_MAT_BIAS_BF16, // explicit HIP-only DS4V fused bias + GGML_OP_RMS_NORM_VISION_F32, // inference-only HIP source-order DS4V normalization + GGML_OP_SOFT_MAX_VISION_F32, // inference-only HIP source-order DS4V softmax + GGML_OP_MUL_MAT_VISION_AV_F32, // inference-only HIP source-layout DS4V AV GGML_OP_DS4_MOE_COMBINE, GGML_OP_COUNT, @@ -1424,6 +1428,28 @@ extern "C" { struct ggml_tensor * a, float eps); + // HIP wave32 only: contiguous F32 [1024, rows], 16 <= rows <= INT_MAX/1024. + // Input values must be BF16-representable. Returns normalized F32 before + // weight multiplication and BF16 rounding; preserves the DS4V source order. + GGML_API struct ggml_tensor * ggml_rms_norm_vision_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // Inference-only HIP wave32 operations; no CPU/RPC/backward implementation. + // scores: contiguous F32, width 16..4096, positive rows, total bytes <= INT_MAX. + GGML_API struct ggml_tensor * ggml_soft_max_vision_f32( + struct ggml_context * ctx, + struct ggml_tensor * scores); + + // v: contiguous F32 [64,16,N,1], probabilities: contiguous F32 [N,N,16,1]. + // N is 16..4096; result is F32 [64,N,16,1]. + GGML_API struct ggml_tensor * ggml_mul_mat_vision_av_f32( + struct ggml_context * ctx, + struct ggml_tensor * v, + struct ggml_tensor * probabilities); + + // group normalize along ne0*ne1*n_groups // used in stable-diffusion GGML_API struct ggml_tensor * ggml_group_norm( @@ -1461,6 +1487,14 @@ extern "C" { // A: k columns, n rows => [ne03, ne02, n, k] // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] + // Explicit inference-only BF16 W[k,m], X[k,n], optional bias[m] -> BF16 Y[m,n]. + // Null bias selects the default Lt epilogue without a bias pointer. A present + // bias, including an all-zero vector, selects the fused bias epilogue. + // Contiguous 2D operands only; HIP Lt capability required, no fallback. + GGML_API struct ggml_tensor * ggml_mul_mat_bias_bf16( + struct ggml_context * ctx, struct ggml_tensor * weight, + struct ggml_tensor * input, struct ggml_tensor * bias); + GGML_API struct ggml_tensor * ggml_mul_mat( struct ggml_context * ctx, struct ggml_tensor * a, diff --git a/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.c b/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.c index ec2690327..242760e18 100644 --- a/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.c +++ b/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.c @@ -1292,3 +1292,248 @@ bool rocmfpx_validate_row_data_fp8(const void * data, size_t nbytes) { return true; } + +static float rocmfpx_bf16_to_fp32(uint16_t v) { + uint32_t bits = (uint32_t) v << 16; + float out; + memcpy(&out, &bits, sizeof(out)); + return out; +} + +static int rocmfpx_mix_nearest_level(float value, const float * levels, int nlevels) { + int lo = 0; + int hi = nlevels - 1; + while (lo < hi) { + const int mid = lo + (hi - lo) / 2; + const float split = 0.5f * (levels[mid] + levels[mid + 1]); + if (value <= split) { + hi = mid; + } else { + lo = mid + 1; + } + } + return lo; +} + +static bool rocmfpx_mix_prepare_books( + const uint16_t * codebooks_bf16, int k, float books[2][8]) { + if (!codebooks_bf16 || k <= 0 || k > 8) { + return false; + } + for (int b = 0; b < 2; ++b) { + for (int i = 0; i < k; ++i) { + const float value = rocmfpx_bf16_to_fp32(codebooks_bf16[b*k + i]); + if (!isfinite(value) || (i > 0 && !(value > books[b][i - 1]))) { + return false; + } + books[b][i] = value; + } + } + return true; +} + +static float rocmfpx_mix_half_error( + const float * x, const float * imatrix, const float * book, int k, + uint8_t e, float best) { + const float scale = rocmfpx_ue4m3_to_fp32(e); + if (!(scale > 0.0f)) { + return INFINITY; + } + const float inv_scale = 1.0f / scale; + float error = 0.0f; + for (int i = 0; i < 16; ++i) { + if (!isfinite(x[i])) { + return INFINITY; + } + const int code = rocmfpx_mix_nearest_level(x[i] * inv_scale, book, k); + const float delta = x[i] - scale * book[code]; + float weight = 1.0f; + if (imatrix) { + if (!isfinite(imatrix[i]) || imatrix[i] < 0.0f) { + return INFINITY; + } + weight = imatrix[i]; + } + error += weight * delta * delta; + if (error > best) { + break; + } + } + return error; +} + +static bool rocmfpx_mix_choose_half( + const float * x, const float * imatrix, float books[2][8], int k, + int * out_book, uint8_t * out_e) { + float max_abs = 0.0f; + for (int i = 0; i < 16; ++i) { + if (!isfinite(x[i])) { + return false; + } + const float ax = fabsf(x[i]); + if (ax > max_abs) { + max_abs = ax; + } + } + if (max_abs == 0.0f) { + *out_book = 0; + *out_e = 0; + return true; + } + + float best_error = INFINITY; + int best_book = 0; + uint8_t best_e = 1; + for (int book = 0; book < 2; ++book) { + float max_level = 0.0f; + for (int i = 0; i < k; ++i) { + const float a = fabsf(books[book][i]); + if (a > max_level) { + max_level = a; + } + } + if (!(max_level > 0.0f)) { + continue; + } + const uint8_t center = rocmfpx_nearest_scale_ue4m3(max_abs / max_level); + for (int delta = -2; delta <= 2; ++delta) { + const int candidate = (int) center + delta; + if (candidate < 1 || candidate > 0x7e) { + continue; + } + const float error = rocmfpx_mix_half_error( + x, imatrix, books[book], k, (uint8_t) candidate, best_error); + if (error < best_error || + (error == best_error && (book < best_book || + (book == best_book && candidate < best_e)))) { + best_error = error; + best_book = book; + best_e = (uint8_t) candidate; + } + } + } + if (!isfinite(best_error)) { + return false; + } + *out_book = best_book; + *out_e = best_e; + return true; +} + +bool rocmfpx_quantize_row_fp2_mix_ref( + const float * GGML_RESTRICT x, block_rocmfp2 * GGML_RESTRICT y, + int64_t k, const uint16_t codebooks_bf16[8], + const float * GGML_RESTRICT imatrix) { + if (!x || !y || k < 0 || k % QK_ROCMFP2 != 0) { + return false; + } + float books[2][8] = {{0}}; + if (!rocmfpx_mix_prepare_books(codebooks_bf16, 4, books)) { + return false; + } + const int64_t nb = k / QK_ROCMFP2; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP2; + block_rocmfp2 * yb = y + ib; + memset(yb, 0, sizeof(*yb)); + for (int half = 0; half < 2; ++half) { + const int off = half*16; + int book = 0; + uint8_t e = 0; + if (!rocmfpx_mix_choose_half( + xb + off, imatrix ? imatrix + ib*QK_ROCMFP2 + off : NULL, + books, 4, &book, &e)) { + return false; + } + yb->e[half] = (uint8_t) (e | (book << 7)); + const float scale = rocmfpx_ue4m3_to_fp32(e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + for (int j = 0; j < 16; ++j) { + const int i = off + j; + const int code = scale > 0.0f + ? rocmfpx_mix_nearest_level(xb[i]*inv_scale, books[book], 4) + : rocmfpx_mix_nearest_level(0.0f, books[book], 4); + yb->qs[i >> 2] |= (uint8_t) (code << (2*(i & 3))); + } + } + } + return true; +} + +void rocmfpx_dequantize_row_fp2_mix( + const block_rocmfp2 * GGML_RESTRICT x, float * GGML_RESTRICT y, + int64_t k, const uint16_t codebooks_bf16[8]) { + assert(x && y && k % QK_ROCMFP2 == 0); + float books[2][8] = {{0}}; + const bool valid_books = rocmfpx_mix_prepare_books(codebooks_bf16, 4, books); + assert(valid_books); + if (!valid_books) return; + for (int64_t ib = 0; ib < k/QK_ROCMFP2; ++ib) { + for (int i = 0; i < QK_ROCMFP2; ++i) { + const uint8_t meta = x[ib].e[i >= 16]; + const int book = meta >> 7; + const float scale = rocmfpx_ue4m3_to_fp32(meta & 0x7f); + const int code = (x[ib].qs[i >> 2] >> (2*(i & 3))) & 3; + y[ib*QK_ROCMFP2 + i] = scale * books[book][code]; + } + } +} + +bool rocmfpx_quantize_row_fp3_mix_ref( + const float * GGML_RESTRICT x, block_rocmfp3 * GGML_RESTRICT y, + int64_t k, const uint16_t codebooks_bf16[16], + const float * GGML_RESTRICT imatrix) { + if (!x || !y || k < 0 || k % QK_ROCMFP3 != 0) { + return false; + } + float books[2][8] = {{0}}; + if (!rocmfpx_mix_prepare_books(codebooks_bf16, 8, books)) { + return false; + } + const int64_t nb = k / QK_ROCMFP3; + for (int64_t ib = 0; ib < nb; ++ib) { + const float * xb = x + ib*QK_ROCMFP3; + block_rocmfp3 * yb = y + ib; + memset(yb, 0, sizeof(*yb)); + for (int half = 0; half < 2; ++half) { + const int off = half*16; + int book = 0; + uint8_t e = 0; + if (!rocmfpx_mix_choose_half( + xb + off, imatrix ? imatrix + ib*QK_ROCMFP3 + off : NULL, + books, 8, &book, &e)) { + return false; + } + yb->e[half] = (uint8_t) (e | (book << 7)); + const float scale = rocmfpx_ue4m3_to_fp32(e); + const float inv_scale = scale > 0.0f ? 1.0f / scale : 0.0f; + for (int j = 0; j < 16; ++j) { + const int i = off + j; + const int code = scale > 0.0f + ? rocmfpx_mix_nearest_level(xb[i]*inv_scale, books[book], 8) + : rocmfpx_mix_nearest_level(0.0f, books[book], 8); + rocmfpx_set_bits(yb->qs, i*3, 3, (uint32_t) code); + } + } + } + return true; +} + +void rocmfpx_dequantize_row_fp3_mix( + const block_rocmfp3 * GGML_RESTRICT x, float * GGML_RESTRICT y, + int64_t k, const uint16_t codebooks_bf16[16]) { + assert(x && y && k % QK_ROCMFP3 == 0); + float books[2][8] = {{0}}; + const bool valid_books = rocmfpx_mix_prepare_books(codebooks_bf16, 8, books); + assert(valid_books); + if (!valid_books) return; + for (int64_t ib = 0; ib < k/QK_ROCMFP3; ++ib) { + for (int i = 0; i < QK_ROCMFP3; ++i) { + const uint8_t meta = x[ib].e[i >= 16]; + const int book = meta >> 7; + const float scale = rocmfpx_ue4m3_to_fp32(meta & 0x7f); + const int code = (int) rocmfpx_get_bits(x[ib].qs, i*3, 3); + y[ib*QK_ROCMFP3 + i] = scale * books[book][code]; + } + } +} diff --git a/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.h b/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.h index f9d6cb896..f202e373d 100644 --- a/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.h +++ b/server/deps/llama.cpp/ggml/rocmfpx/rocmfpx.h @@ -86,11 +86,20 @@ GGML_API void rocmfpx_dequantize_row_fp2(const block_rocmfp2 * GGML_RESTRICT x GGML_API void rocmfpx_quantize_row_fp2(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); GGML_API size_t rocmfpx_quantize_fp2(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +// Adaptive MIX block codecs. `codebooks_bf16` contains two sorted codebooks +// (2*4 or 2*8 levels). The high bit of each half-block metadata byte selects +// the codebook; the low seven bits retain the unsigned E4M3 scale. +GGML_API bool rocmfpx_quantize_row_fp2_mix_ref(const float * GGML_RESTRICT x, block_rocmfp2 * GGML_RESTRICT y, int64_t k, const uint16_t codebooks_bf16[8], const float * GGML_RESTRICT imatrix); +GGML_API void rocmfpx_dequantize_row_fp2_mix(const block_rocmfp2 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const uint16_t codebooks_bf16[8]); + GGML_API void rocmfpx_quantize_row_fp3_ref(const float * GGML_RESTRICT x, block_rocmfp3 * GGML_RESTRICT y, int64_t k); GGML_API void rocmfpx_dequantize_row_fp3(const block_rocmfp3 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void rocmfpx_quantize_row_fp3(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); GGML_API size_t rocmfpx_quantize_fp3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API bool rocmfpx_quantize_row_fp3_mix_ref(const float * GGML_RESTRICT x, block_rocmfp3 * GGML_RESTRICT y, int64_t k, const uint16_t codebooks_bf16[16], const float * GGML_RESTRICT imatrix); +GGML_API void rocmfpx_dequantize_row_fp3_mix(const block_rocmfp3 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const uint16_t codebooks_bf16[16]); + GGML_API void rocmfpx_quantize_row_fp6_ref(const float * GGML_RESTRICT x, block_rocmfp6 * GGML_RESTRICT y, int64_t k); GGML_API void rocmfpx_dequantize_row_fp6(const block_rocmfp6 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void rocmfpx_quantize_row_fp6(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); diff --git a/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c b/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c index aeed24182..415ee2197 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c +++ b/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c @@ -2244,6 +2244,13 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { GGML_ABORT("GGML_OP_FLASH_ATTN_SPARSE is only supported on the CUDA backend"); } + case GGML_OP_MUL_MAT_BIAS_BF16: + { GGML_ABORT("GGML_OP_MUL_MAT_BIAS_BF16 requires the HIP Lt backend"); } + case GGML_OP_RMS_NORM_VISION_F32: + { GGML_ABORT("GGML_OP_RMS_NORM_VISION_F32 requires the HIP wave32 backend"); } + case GGML_OP_SOFT_MAX_VISION_F32: + case GGML_OP_MUL_MAT_VISION_AV_F32: + { GGML_ABORT("explicit vision attention requires the HIP wave32 backend"); } case GGML_OP_PAGED_ATTN: { GGML_ABORT("GGML_OP_PAGED_ATTN is only supported on the CUDA backend"); @@ -2646,6 +2653,13 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { { n_tasks = n_threads; } break; + case GGML_OP_MUL_MAT_BIAS_BF16: + GGML_ABORT("GGML_OP_MUL_MAT_BIAS_BF16 cannot be planned on CPU"); + case GGML_OP_RMS_NORM_VISION_F32: + GGML_ABORT("GGML_OP_RMS_NORM_VISION_F32 cannot be planned on CPU"); + case GGML_OP_SOFT_MAX_VISION_F32: + case GGML_OP_MUL_MAT_VISION_AV_F32: + GGML_ABORT("explicit vision attention cannot be planned on CPU"); case GGML_OP_RWKV_WKV6: case GGML_OP_GATED_LINEAR_ATTN: case GGML_OP_RWKV_WKV7: diff --git a/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp b/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp index 8e94d05b4..2b3501eca 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/server/deps/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -421,6 +421,9 @@ static ggml_backend_buffer_t ggml_backend_cpu_device_buffer_from_host_ptr(ggml_b } static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + // No extra-buffer handler may accidentally advertise this HIP-only op. + if (op->op == GGML_OP_MUL_MAT_BIAS_BF16 || op->op == GGML_OP_RMS_NORM_VISION_F32 || + op->op == GGML_OP_SOFT_MAX_VISION_F32 || op->op == GGML_OP_MUL_MAT_VISION_AV_F32) return false; const struct ggml_tensor * src0 = op->src[0]; const struct ggml_tensor * src1 = op->src[1]; @@ -471,6 +474,10 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st // The generic CPU kernel does not implement that contract. return !ggml_flash_attn_ext_is_ds4(op); case GGML_OP_PAGED_ATTN: + case GGML_OP_MUL_MAT_BIAS_BF16: + case GGML_OP_RMS_NORM_VISION_F32: + case GGML_OP_SOFT_MAX_VISION_F32: + case GGML_OP_MUL_MAT_VISION_AV_F32: return false; case GGML_OP_SSM_CONV: // Every nonzero mode is a dflash CUDA/HIP extension (SpecLA, diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh index 7a30664c4..9c11002f5 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/common.cuh @@ -34,6 +34,9 @@ #if defined(GGML_USE_HIP) #include "vendors/hip.h" +#if defined(GGML_HIP_DS4V_VISION) +#include +#endif #elif defined(GGML_USE_MUSA) #include "vendors/musa.h" #else @@ -1448,6 +1451,16 @@ struct ggml_backend_cuda_context { cudaStream_t streams[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = { { nullptr } }; cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr}; +#if defined(GGML_HIP_DS4V_VISION) + hipblasLtHandle_t vision_bias_handle = nullptr; + void * vision_bias_workspace = nullptr; // shared linear/AV 76 MiB, retained until context destruction + cudaEvent_t vision_bias_event = nullptr; + size_t vision_bias_launches = 0; + size_t vision_norm_launches = 0; + size_t vision_softmax_launches = 0; + size_t vision_av_launches = 0; + size_t vision_rotary_launches = 0; // successful synchronous table calls (five kernels each) +#endif int curr_stream_no = 0; bool low_priority_streams = false; int stream_priority = 0; diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu index 20c64afb9..42c292c56 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu @@ -3,6 +3,10 @@ #include "ggml-backend-impl.h" #include "ggml-cuda/common.cuh" +#include "ggml-cuda/vision-bias.cuh" +#include "ggml-cuda/vision-rotary.cuh" +#include "ggml-cuda/vision-softmax.cuh" +#include "ggml-cuda/vision-av.cuh" #include "ggml-cuda/acc.cuh" #include "ggml-cuda/add-id.cuh" #include "ggml-cuda/arange.cuh" @@ -729,6 +733,16 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { luce_q8_memo.pop_back(); } +#if defined(GGML_HIP_DS4V_VISION) + if (vision_bias_workspace) { + ggml_cuda_set_device(device); + // The latest event follows every use of the shared workspace. + if (vision_bias_launches || vision_av_launches) CUDA_CHECK(cudaEventSynchronize(vision_bias_event)); + CUDA_CHECK(cudaFree(vision_bias_workspace)); + CUDA_CHECK(cudaEventDestroy(vision_bias_event)); + } + if (vision_bias_handle) CUBLAS_CHECK(hipblasLtDestroy(vision_bias_handle)); +#endif if (copy_event != nullptr) { CUDA_CHECK(cudaEventDestroy(copy_event)); copy_event = nullptr; @@ -3597,6 +3611,34 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg case GGML_OP_FLASH_ATTN_SPARSE: ggml_cuda_flash_attn_sparse(ctx, dst); break; + case GGML_OP_MUL_MAT_BIAS_BF16: +#if defined(GGML_HIP_DS4V_VISION) + ggml_hip_vision_bias(ctx, dst); + break; +#else + return false; +#endif + case GGML_OP_RMS_NORM_VISION_F32: +#if defined(GGML_HIP_DS4V_VISION) + ggml_hip_vision_norm(ctx, dst); + break; +#else + return false; +#endif + case GGML_OP_SOFT_MAX_VISION_F32: +#if defined(GGML_HIP_DS4V_VISION) + ggml_cuda_op_soft_max_vision_f32(ctx, dst); + break; +#else + return false; +#endif + case GGML_OP_MUL_MAT_VISION_AV_F32: +#if defined(GGML_HIP_DS4V_VISION) + ggml_hip_vision_av_f32(ctx, dst); + break; +#else + return false; +#endif case GGML_OP_PAGED_ATTN: ggml_cuda_paged_attn(ctx, dst); break; @@ -3885,6 +3927,10 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) { for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; + // These explicit vision operations are qualified through direct + // execution only; the linear also retains a workspace and event. + if (node->op == GGML_OP_MUL_MAT_BIAS_BF16 || node->op == GGML_OP_RMS_NORM_VISION_F32 || + node->op == GGML_OP_SOFT_MAX_VISION_F32 || node->op == GGML_OP_MUL_MAT_VISION_AV_F32) return false; if (ggml_is_empty(node) || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE || node->op == GGML_OP_NONE) { continue; @@ -6523,6 +6569,30 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g return ggml_cuda_flash_attn_ext_supported(dev_ctx->device, op); case GGML_OP_FLASH_ATTN_SPARSE: return true; // Always supported on CUDA + case GGML_OP_MUL_MAT_BIAS_BF16: +#if defined(GGML_HIP_DS4V_VISION) + return ggml_hip_vision_bias_supported(op); +#else + return false; +#endif + case GGML_OP_RMS_NORM_VISION_F32: +#if defined(GGML_HIP_DS4V_VISION) + return ggml_hip_vision_norm_supported(dev_ctx->device, op); +#else + return false; +#endif + case GGML_OP_SOFT_MAX_VISION_F32: +#if defined(GGML_HIP_DS4V_VISION) + return ggml_hip_vision_softmax_f32_supported(dev_ctx->device, op); +#else + return false; +#endif + case GGML_OP_MUL_MAT_VISION_AV_F32: +#if defined(GGML_HIP_DS4V_VISION) + return ggml_hip_vision_av_f32_supported(dev_ctx->device, op); +#else + return false; +#endif case GGML_OP_PAGED_ATTN: return ggml_cuda_paged_attn_supported(op); case GGML_OP_CROSS_ENTROPY_LOSS: @@ -6700,7 +6770,66 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t GGML_UNUSED(reg); } +#if defined(GGML_HIP_DS4V_VISION) +static size_t ggml_backend_hip_vision_bias_bf16_workspace(ggml_backend_t backend) { + return backend && ggml_backend_is_cuda(backend) ? 76ULL*1024*1024 : 0; +} +static size_t ggml_backend_hip_vision_bias_bf16_launches(ggml_backend_t backend) { + return backend && ggml_backend_is_cuda(backend) ? + static_cast(backend->context)->vision_bias_launches : 0; +} +static bool ggml_backend_hip_vision_norm_f32_capable(ggml_backend_t backend) { + return backend && ggml_backend_is_cuda(backend) && + ggml_hip_vision_norm_capable(static_cast(backend->context)->device); +} +static size_t ggml_backend_hip_vision_norm_f32_launches(ggml_backend_t backend) { + return backend && ggml_backend_is_cuda(backend) ? + static_cast(backend->context)->vision_norm_launches : 0; +} +static bool ggml_backend_hip_vision_softmax_f32_capable(ggml_backend_t backend) { + return ggml_backend_hip_vision_norm_f32_capable(backend); +} +static size_t ggml_backend_hip_vision_softmax_f32_launches(ggml_backend_t backend) { + return backend && ggml_backend_is_cuda(backend) ? + static_cast(backend->context)->vision_softmax_launches : 0; +} +static bool ggml_backend_hip_vision_av_f32_capable(ggml_backend_t backend) { + return ggml_backend_hip_vision_norm_f32_capable(backend); +} +static size_t ggml_backend_hip_vision_av_f32_launches(ggml_backend_t backend) { + return backend && ggml_backend_is_cuda(backend) ? + static_cast(backend->context)->vision_av_launches : 0; +} +static bool ggml_backend_hip_vision_rotary_f32_capable(ggml_backend_t backend) { + return ggml_backend_hip_vision_norm_f32_capable(backend); +} +static bool ggml_backend_hip_vision_rotary_f32(ggml_backend_t backend, int64_t height, int64_t width, + float * cosine, float * sine) { + return ggml_backend_hip_vision_rotary_f32_capable(backend) && + ggml_hip_vision_rotary(*static_cast(backend->context), + height, width, cosine, sine); +} +static size_t ggml_backend_hip_vision_rotary_f32_launches(ggml_backend_t backend) { + return backend && ggml_backend_is_cuda(backend) ? + static_cast(backend->context)->vision_rotary_launches : 0; +} + +#endif + static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) { +#if defined(GGML_HIP_DS4V_VISION) + if (strcmp(name,"ggml_backend_hip_vision_bias_bf16_workspace")==0) return (void *)ggml_backend_hip_vision_bias_bf16_workspace; + if (strcmp(name,"ggml_backend_hip_vision_bias_bf16_launches")==0) return (void *)ggml_backend_hip_vision_bias_bf16_launches; + if (strcmp(name,"ggml_backend_hip_vision_norm_f32_capable")==0) return (void *)ggml_backend_hip_vision_norm_f32_capable; + if (strcmp(name,"ggml_backend_hip_vision_norm_f32_launches")==0) return (void *)ggml_backend_hip_vision_norm_f32_launches; + if (strcmp(name,"ggml_backend_hip_vision_softmax_f32_capable")==0) return (void *)ggml_backend_hip_vision_softmax_f32_capable; + if (strcmp(name,"ggml_backend_hip_vision_softmax_f32_launches")==0) return (void *)ggml_backend_hip_vision_softmax_f32_launches; + if (strcmp(name,"ggml_backend_hip_vision_av_f32_capable")==0) return (void *)ggml_backend_hip_vision_av_f32_capable; + if (strcmp(name,"ggml_backend_hip_vision_av_f32_launches")==0) return (void *)ggml_backend_hip_vision_av_f32_launches; + if (strcmp(name,"ggml_backend_hip_vision_rotary_f32_capable")==0) return (void *)ggml_backend_hip_vision_rotary_f32_capable; + if (strcmp(name,"ggml_backend_hip_vision_rotary_f32")==0) return (void *)ggml_backend_hip_vision_rotary_f32; + if (strcmp(name,"ggml_backend_hip_vision_rotary_f32_launches")==0) return (void *)ggml_backend_hip_vision_rotary_f32_launches; +#endif GGML_UNUSED(reg); if (strcmp(name, "ggml_backend_comm_init") == 0) { return (void *)ggml_backend_cuda_comm_init; diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/norm.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/norm.cu index 696a6f441..5e01ae0a0 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/norm.cu +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/norm.cu @@ -1,6 +1,70 @@ #include "norm.cuh" +#include +#include #include +#if defined(GGML_HIP_DS4V_VISION) +bool ggml_hip_vision_norm_capable(int device) { + const auto & info = ggml_cuda_info(); + return device >= 0 && device < info.device_count && info.devices[device].warp_size == 32; +} + +bool ggml_hip_vision_norm_supported(int device, const ggml_tensor * op) { + if (!ggml_hip_vision_norm_capable(device) || !op || op->op != GGML_OP_RMS_NORM_VISION_F32 || + !op->src[0] || op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || + !ggml_is_contiguous(op) || !ggml_is_contiguous(op->src[0]) || + !ggml_are_same_shape(op, op->src[0]) || op->ne[0] != 1024 || op->ne[1] < 16 || + op->ne[1] > INT_MAX/1024 || op->ne[2] != 1 || op->ne[3] != 1) { + return false; + } + float eps; + memcpy(&eps, op->op_params, sizeof(eps)); + return std::isfinite(eps) && eps >= 0.0f; +} + +// PyTorch 3d3aa833db84eed6b7f5595cb5f162c2f78300a4, aten/src/ATen/native/cuda/ +// Reduce.cuh:499-557,655-668: four input accumulators and ascending ROCm shuffles. +// With contiguous width1024 and >=16 rows the source uses one wave32 per row. +static __global__ void rms_norm_vision_f32(const float * x, float * dst, int rows, float eps) { + const int row = blockIdx.x * 16 + threadIdx.y; + const int lane = threadIdx.x; + if (row >= rows) { + return; + } + float a[4] = {0, 0, 0, 0}; + for (int k = 0; k < 8; ++k) { + for (int j = 0; j < 4; ++j) { + const float xi = x[row*1024 + 4*(lane + 32*k) + j]; + volatile float squared = xi*xi; // source materializes F32 square before mean + a[j] = a[j] + squared; + } + } + float sum = ((a[0] + a[1]) + a[2]) + a[3]; + for (int offset = 1; offset < 32; offset *= 2) { + sum = sum + __shfl_down(sum, offset, 32); + } + sum = __shfl(sum, 0, 32); + const float mean = sum * (1.0f/1024.0f); + // Same pinned Torch UnaryOpsKernel.cu:78-80 calls global ::rsqrt. Its ROCm + // overload promotes float to double; keep the verified promotion explicitly. + const float scale = static_cast(::rsqrt(static_cast(mean + eps))); + for (int col = lane; col < 1024; col += 32) { + dst[row*1024 + col] = x[row*1024 + col] * scale; + } +} + +void ggml_hip_vision_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + GGML_ASSERT(ggml_hip_vision_norm_supported(ctx.device, dst)); + float eps; + memcpy(&eps, dst->op_params, sizeof(eps)); + const int rows = static_cast(dst->ne[1]); + rms_norm_vision_f32<<<(rows + 15)/16, dim3(32, 16), 0, ctx.stream()>>>( + static_cast(dst->src[0]->data), static_cast(dst->data), rows, eps); + CUDA_CHECK(cudaGetLastError()); + ++ctx.vision_norm_launches; +} +#endif + template static __global__ void norm_f32( const float * x, float * dst, const int ncols, const int64_t stride_row, const int64_t stride_channel, diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/norm.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/norm.cuh index 6313a98ce..b0eb27e2c 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-cuda/norm.cuh +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/norm.cuh @@ -6,6 +6,12 @@ void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +#if defined(GGML_HIP_DS4V_VISION) +bool ggml_hip_vision_norm_capable(int device); +bool ggml_hip_vision_norm_supported(int device, const ggml_tensor * op); +void ggml_hip_vision_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +#endif + void ggml_cuda_op_rms_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor); void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-av.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-av.cu new file mode 100644 index 000000000..12cbfc266 --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-av.cu @@ -0,0 +1,74 @@ +#include "vision-av.cuh" +#include "vision-bias.cuh" +#include "norm.cuh" + +#if defined(GGML_HIP_DS4V_VISION) +bool ggml_hip_vision_av_f32_supported(int device, const ggml_tensor * op) { + if (!ggml_hip_vision_norm_capable(device) || !op || op->op != GGML_OP_MUL_MAT_VISION_AV_F32 || + !op->src[0] || !op->src[1] || op->type != GGML_TYPE_F32) { + return false; + } + for (int i = 2; i < GGML_MAX_SRC; ++i) { + if (op->src[i]) return false; + } + const auto v = op->src[0], p = op->src[1]; + if (v->type != GGML_TYPE_F32 || p->type != GGML_TYPE_F32 || + v->ne[0] != 64 || v->ne[1] != 16 || v->ne[3] != 1) { + return false; + } + const int64_t n = v->ne[2]; + return n >= 16 && n <= 4096 && p->ne[0] == n && p->ne[1] == n && p->ne[2] == 16 && p->ne[3] == 1 && + op->ne[0] == 64 && op->ne[1] == n && op->ne[2] == 16 && op->ne[3] == 1 && + ggml_is_contiguous(v) && ggml_is_contiguous(p) && ggml_is_contiguous(op); +} + +void ggml_hip_vision_av_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + GGML_ASSERT(ggml_hip_vision_av_f32_supported(ctx.device, dst)); + ggml_cuda_set_device(ctx.device); + const auto stream = ctx.stream(); + cudaStreamCaptureStatus capture; + CUDA_CHECK(cudaStreamIsCapturing(stream, &capture)); + GGML_ASSERT(capture == cudaStreamCaptureStatusNone && "HIP vision AV requires direct execution"); + ggml_hip_vision_workspace_acquire(ctx); + const int64_t n = dst->src[0]->ne[2]; + hipblasLtMatmulDesc_t op = nullptr; + hipblasLtMatrixLayout_t a = nullptr, b = nullptr, c = nullptr; + hipblasLtMatmulPreference_t pref = nullptr; + CUBLAS_CHECK(hipblasLtMatmulDescCreate(&op,HIPBLAS_COMPUTE_32F,HIP_R_32F)); + const hipblasOperation_t transpose = HIPBLAS_OP_N; + CUBLAS_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSA,&transpose,sizeof(transpose))); + CUBLAS_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSB,&transpose,sizeof(transpose))); + // Pinned Torch 3d3aa833 Blas.cpp/CUDABlas.cpp: row-major output swaps the + // operands, retaining V's token-major stride. Default epilogue, C equals D. + CUBLAS_CHECK(hipblasLtMatrixLayoutCreate(&a,HIP_R_32F,64,n,1024)); + CUBLAS_CHECK(hipblasLtMatrixLayoutCreate(&b,HIP_R_32F,n,n,n)); + CUBLAS_CHECK(hipblasLtMatrixLayoutCreate(&c,HIP_R_32F,64,n,64)); + const int batches = 16; + const int64_t stride_a = 64, stride_b = n*n, stride_c = n*64; + for (auto layout : {a,b,c}) { + CUBLAS_CHECK(hipblasLtMatrixLayoutSetAttribute(layout,HIPBLASLT_MATRIX_LAYOUT_BATCH_COUNT,&batches,sizeof(batches))); + } + CUBLAS_CHECK(hipblasLtMatrixLayoutSetAttribute(a,HIPBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET,&stride_a,sizeof(stride_a))); + CUBLAS_CHECK(hipblasLtMatrixLayoutSetAttribute(b,HIPBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET,&stride_b,sizeof(stride_b))); + CUBLAS_CHECK(hipblasLtMatrixLayoutSetAttribute(c,HIPBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET,&stride_c,sizeof(stride_c))); + CUBLAS_CHECK(hipblasLtMatmulPreferenceCreate(&pref)); + constexpr size_t bytes = GGML_HIP_VISION_WORKSPACE_BYTES; + CUBLAS_CHECK(hipblasLtMatmulPreferenceSetAttribute(pref,HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES,&bytes,sizeof(bytes))); + hipblasLtMatmulHeuristicResult_t heuristic{}; + int returned = 0; + CUBLAS_CHECK(hipblasLtMatmulAlgoGetHeuristic(ctx.vision_bias_handle,op,a,b,c,c,pref,1,&heuristic,&returned)); + GGML_ASSERT(returned == 1 && "HIP vision AV: no first Lt heuristic; fallback forbidden"); + CUBLAS_CHECK(heuristic.state); + GGML_ASSERT(heuristic.workspaceSize <= bytes); + const float alpha = 1.0f, beta = 0.0f; + CUBLAS_CHECK(hipblasLtMatmul(ctx.vision_bias_handle,op,&alpha,dst->src[0]->data,a,dst->src[1]->data,b, + &beta,dst->data,c,dst->data,c,&heuristic.algo,ctx.vision_bias_workspace,bytes,stream)); + ggml_hip_vision_workspace_record(ctx); + ++ctx.vision_av_launches; + CUBLAS_CHECK(hipblasLtMatmulPreferenceDestroy(pref)); + CUBLAS_CHECK(hipblasLtMatrixLayoutDestroy(c)); + CUBLAS_CHECK(hipblasLtMatrixLayoutDestroy(b)); + CUBLAS_CHECK(hipblasLtMatrixLayoutDestroy(a)); + CUBLAS_CHECK(hipblasLtMatmulDescDestroy(op)); +} +#endif diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-av.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-av.cuh new file mode 100644 index 000000000..efc14fb61 --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-av.cuh @@ -0,0 +1,7 @@ +#pragma once +#include "common.cuh" + +#if defined(GGML_HIP_DS4V_VISION) +bool ggml_hip_vision_av_f32_supported(int device, const ggml_tensor * op); +void ggml_hip_vision_av_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +#endif diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-bias.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-bias.cu new file mode 100644 index 000000000..d0be88183 --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-bias.cu @@ -0,0 +1,72 @@ +#include "vision-bias.cuh" +#if defined(GGML_HIP_DS4V_VISION) +#include + +bool ggml_hip_vision_bias_supported(const ggml_tensor * d) { + if (!d || d->type != GGML_TYPE_BF16 || !ggml_is_contiguous(d)) return false; + const auto w=d->src[0],x=d->src[1],b=d->src[2]; + if (!w || !x) return false; + for (auto t : {w,x}) if (t->type != GGML_TYPE_BF16 || !ggml_is_contiguous(t) || t->ne[2]!=1 || t->ne[3]!=1) return false; + if (b && (b->type!=GGML_TYPE_BF16 || !ggml_is_contiguous(b) || !ggml_is_vector(b) || b->ne[0]!=w->ne[1])) return false; + return w->ne[0]>0 && w->ne[0]<=INT_MAX && w->ne[1]>0 && w->ne[1]<=INT_MAX && + x->ne[1]>0 && x->ne[1]<=INT_MAX && x->ne[0]==w->ne[0] && + d->ne[0]==w->ne[1] && d->ne[1]==x->ne[1] && d->ne[2]==1 && d->ne[3]==1; +} + +void ggml_hip_vision_workspace_acquire(ggml_backend_cuda_context &ctx) { + ggml_cuda_set_device(ctx.device); + // A single event serializes shared workspace use across context streams. + if (!ctx.vision_bias_handle) CUBLAS_CHECK(hipblasLtCreate(&ctx.vision_bias_handle)); + if (!ctx.vision_bias_workspace) { + CUDA_CHECK(cudaMalloc(&ctx.vision_bias_workspace,GGML_HIP_VISION_WORKSPACE_BYTES)); + CUDA_CHECK(cudaEventCreateWithFlags(&ctx.vision_bias_event,cudaEventDisableTiming)); + } + if (ctx.vision_bias_launches || ctx.vision_av_launches) { + CUDA_CHECK(cudaStreamWaitEvent(ctx.stream(),ctx.vision_bias_event,0)); + } +} + +void ggml_hip_vision_workspace_record(ggml_backend_cuda_context &ctx) { + CUDA_CHECK(cudaEventRecord(ctx.vision_bias_event,ctx.stream())); +} + +void ggml_hip_vision_bias(ggml_backend_cuda_context &ctx, ggml_tensor *dst) { + GGML_ASSERT(ggml_hip_vision_bias_supported(dst)); + ggml_cuda_set_device(ctx.device); + const auto stream=ctx.stream(); + constexpr size_t bytes=GGML_HIP_VISION_WORKSPACE_BYTES; + ggml_hip_vision_workspace_acquire(ctx); + const auto w=dst->src[0],x=dst->src[1],b=dst->src[2]; + const int64_t k=w->ne[0],m=w->ne[1],n=x->ne[1]; + hipblasLtMatmulDesc_t op=nullptr; + hipblasLtMatrixLayout_t a=nullptr,bl=nullptr,c=nullptr; + hipblasLtMatmulPreference_t pref=nullptr; + CUBLAS_CHECK(hipblasLtMatmulDescCreate(&op,HIPBLAS_COMPUTE_32F,HIP_R_32F)); + const hipblasOperation_t ta=HIPBLAS_OP_T,tb=HIPBLAS_OP_N; + const hipblasLtEpilogue_t epilogue=b ? HIPBLASLT_EPILOGUE_BIAS : HIPBLASLT_EPILOGUE_DEFAULT; + CUBLAS_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSA,&ta,sizeof(ta))); + CUBLAS_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_TRANSB,&tb,sizeof(tb))); + CUBLAS_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_EPILOGUE,&epilogue,sizeof(epilogue))); + if (b) CUBLAS_CHECK(hipblasLtMatmulDescSetAttribute(op,HIPBLASLT_MATMUL_DESC_BIAS_POINTER,&b->data,sizeof(b->data))); + CUBLAS_CHECK(hipblasLtMatrixLayoutCreate(&a,HIP_R_16BF,k,m,k)); + CUBLAS_CHECK(hipblasLtMatrixLayoutCreate(&bl,HIP_R_16BF,k,n,k)); + CUBLAS_CHECK(hipblasLtMatrixLayoutCreate(&c,HIP_R_16BF,m,n,m)); + CUBLAS_CHECK(hipblasLtMatmulPreferenceCreate(&pref)); + CUBLAS_CHECK(hipblasLtMatmulPreferenceSetAttribute(pref,HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES,&bytes,sizeof(bytes))); + hipblasLtMatmulHeuristicResult_t heuristic{}; int returned=0; + CUBLAS_CHECK(hipblasLtMatmulAlgoGetHeuristic(ctx.vision_bias_handle,op,a,bl,c,c,pref,1,&heuristic,&returned)); + GGML_ASSERT(returned==1 && "HIP vision bias: no first Lt heuristic; fallback forbidden"); + CUBLAS_CHECK(heuristic.state); + GGML_ASSERT(heuristic.workspaceSize<=bytes); + const float alpha=1,beta=0; + CUBLAS_CHECK(hipblasLtMatmul(ctx.vision_bias_handle,op,&alpha,w->data,a,x->data,bl,&beta, + dst->data,c,dst->data,c,&heuristic.algo,ctx.vision_bias_workspace,bytes,stream)); + ggml_hip_vision_workspace_record(ctx); + ++ctx.vision_bias_launches; + CUBLAS_CHECK(hipblasLtMatmulPreferenceDestroy(pref)); + CUBLAS_CHECK(hipblasLtMatrixLayoutDestroy(c)); + CUBLAS_CHECK(hipblasLtMatrixLayoutDestroy(bl)); + CUBLAS_CHECK(hipblasLtMatrixLayoutDestroy(a)); + CUBLAS_CHECK(hipblasLtMatmulDescDestroy(op)); +} +#endif diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-bias.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-bias.cuh new file mode 100644 index 000000000..ad3a233f7 --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-bias.cuh @@ -0,0 +1,11 @@ +#pragma once +#include "common.cuh" +#if defined(GGML_HIP_DS4V_VISION) +// One workspace/handle/event shared by explicit linear and AV operations. +constexpr size_t GGML_HIP_VISION_WORKSPACE_BYTES = 76ULL*1024*1024; +void ggml_hip_vision_workspace_acquire(ggml_backend_cuda_context & ctx); +void ggml_hip_vision_workspace_record(ggml_backend_cuda_context & ctx); +// DS4V-only explicit op; no ordinary MUL_MAT/ADD fusion or fallback. +bool ggml_hip_vision_bias_supported(const ggml_tensor * dst); +void ggml_hip_vision_bias(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +#endif diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-rotary.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-rotary.cu new file mode 100644 index 000000000..b36a705bf --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-rotary.cu @@ -0,0 +1,117 @@ +#include "vision-rotary.cuh" +#include "norm.cuh" +#include +#include + +#if defined(GGML_HIP_DS4V_VISION) +// PyTorch 3d3aa833db84eed6b7f5595cb5f162c2f78300a4, aten/src/ATen/native/cuda/: +// Pow.cuh uses global ::pow on Linux; UnaryFractionKernels.cu uses float 1/a; +// UnaryGeometric{Cos,Sin}Kernel.cu use global ::cos and ::sin. The five kernels +// preserve eager F32 materialization between power, reciprocal, angle and trig. +static __global__ void vision_rotary_power(float * output, float base) { + static_assert(sizeof(decltype(::pow(0.0f, 0.0f))) == sizeof(float), "source pow overload changed"); + const int j = threadIdx.x; + if (j < 16) { + output[j] = ::pow(base, float(2*j)*(1.0f/32.0f)); + } +} + +static __global__ void vision_rotary_inverse(const float * power, float * inverse) { + const int j = threadIdx.x; + if (j < 16) { + inverse[j] = 1.0f/power[j]; + } +} + +static __global__ void vision_rotary_angle(const float * inverse, float * angle, int count, int width) { + const int i = blockIdx.x*blockDim.x + threadIdx.x; + if (i < count) { + const int row = i/32, j = i%32; + angle[i] = float(j < 16 ? row/width : row%width)*inverse[j%16]; + } +} + +static __global__ void vision_rotary_cos(const float * angle, float * cosine, int count) { + static_assert(sizeof(decltype(::cos(0.0f))) == sizeof(float), "source cos overload changed"); + const int i = blockIdx.x*blockDim.x + threadIdx.x; + if (i < count) { + cosine[i] = ::cos(angle[i]); + } +} + +static __global__ void vision_rotary_sin(const float * angle, float * sine, int count) { + static_assert(sizeof(decltype(::sin(0.0f))) == sizeof(float), "source sin overload changed"); + const int i = blockIdx.x*blockDim.x + threadIdx.x; + if (i < count) { + sine[i] = ::sin(angle[i]); + } +} + +namespace { +struct rotary_temporary { + float * data = nullptr; + cudaStream_t stream; + + rotary_temporary(size_t bytes, cudaStream_t stream) : stream(stream) { + CUDA_CHECK(cudaMalloc(reinterpret_cast(&data), bytes)); + } + ~rotary_temporary() { + CUDA_CHECK(cudaStreamSynchronize(stream)); + CUDA_CHECK(cudaFree(data)); + } + rotary_temporary(const rotary_temporary &) = delete; + rotary_temporary & operator=(const rotary_temporary &) = delete; +}; +} // namespace + +bool ggml_hip_vision_rotary(ggml_backend_cuda_context & ctx, int64_t height, int64_t width, + float * cosine, float * sine) { + if (!ggml_hip_vision_norm_capable(ctx.device) || !cosine || !sine || cosine == sine || + height <= 0 || width <= 0 || height > 1152 || width > 1152) { + return false; + } + const size_t h = static_cast(height), w = static_cast(width); + constexpr size_t cap = 128ULL*1024*1024; + constexpr size_t workspace = 76ULL*1024*1024; + constexpr size_t fixed_bytes = 2*16*sizeof(float); + constexpr size_t bytes_per_row = 3*32*sizeof(float); + if (h > std::numeric_limits::max()/w) { + return false; + } + const size_t rows = h*w; + if (rows > (cap - workspace - fixed_bytes)/bytes_per_row) { + return false; + } + const size_t bytes = fixed_bytes + rows*bytes_per_row; + const int count = static_cast(rows*32); + ggml_cuda_set_device(ctx.device); + const auto stream = ctx.stream(); + cudaStreamCaptureStatus capture; + CUDA_CHECK(cudaStreamIsCapturing(stream, &capture)); + if (capture != cudaStreamCaptureStatusNone) { + return false; + } + { + rotary_temporary temporary(bytes, stream); + float * power = temporary.data; + float * inverse = power + 16; + float * angle = inverse + 16; + float * cos = angle + count; + float * sin = cos + count; + vision_rotary_power<<<1, 32, 0, stream>>>(power, 10000.0f); + CUDA_CHECK(cudaGetLastError()); + vision_rotary_inverse<<<1, 32, 0, stream>>>(power, inverse); + CUDA_CHECK(cudaGetLastError()); + vision_rotary_angle<<<(count + 255)/256, 256, 0, stream>>>(inverse, angle, count, int(width)); + CUDA_CHECK(cudaGetLastError()); + vision_rotary_cos<<<(count + 255)/256, 256, 0, stream>>>(angle, cos, count); + CUDA_CHECK(cudaGetLastError()); + vision_rotary_sin<<<(count + 255)/256, 256, 0, stream>>>(angle, sin, count); + CUDA_CHECK(cudaGetLastError()); + CUDA_CHECK(cudaMemcpyAsync(cosine, cos, count*sizeof(float), cudaMemcpyDeviceToHost, stream)); + CUDA_CHECK(cudaMemcpyAsync(sine, sin, count*sizeof(float), cudaMemcpyDeviceToHost, stream)); + } + ++ctx.vision_rotary_launches; + return true; +} +#endif diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-rotary.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-rotary.cuh new file mode 100644 index 000000000..bc3c62f6d --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-rotary.cuh @@ -0,0 +1,9 @@ +#pragma once +#include "common.cuh" + +#if defined(GGML_HIP_DS4V_VISION) +// Caller supplies two distinct host buffers of height*width*32 floats each. +// Success fills both buffers synchronously and records one complete table call. +bool ggml_hip_vision_rotary(ggml_backend_cuda_context & ctx, int64_t height, int64_t width, + float * cosine, float * sine); +#endif diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-softmax-kernels.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-softmax-kernels.cuh new file mode 100644 index 000000000..4c416fc40 --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-softmax-kernels.cuh @@ -0,0 +1,318 @@ +#pragma once + +// F32, wave32 adaptation of PyTorch 3d3aa833db84eed6b7f5595cb5f162c2f78300a4: +// aten/src/ATen/native/cuda/{PersistentSoftmax.cuh,SoftMax.cu,block_reduce.cuh}. +// Preserve source traversal, reduction, exp, and division order. Build without +// fast math or floating-point contraction. See the full upstream license below. +#include +#include +#include +#include +#include + +namespace ggml_vision_softmax { + +template +static __global__ void persistent(const float * input, float * output, int width, int rows) { + constexpr int power = 1 << LOG2_WIDTH; + constexpr int lanes = power < 32 ? power : 32; + constexpr int iterations = power / lanes; + constexpr int batch = power <= 128 ? 2 : 1; + const int first = (int(blockDim.y) * int(blockIdx.x) + int(threadIdx.y)) * batch; + const int lane = int(threadIdx.x); + float values[batch][iterations]; + float maximum[batch]; + float sums[batch]; + static_assert(std::is_same::value, "float exp required"); +#pragma unroll + for (int b = 0; b < batch; ++b) { +#pragma unroll + for (int it = 0; it < iterations; ++it) { + const int col = lane + it * lanes; + values[b][it] = first + b < rows && col < width ? + input[(first + b) * width + col] : -std::numeric_limits::infinity(); + } + maximum[b] = values[b][0]; +#pragma unroll + for (int it = 0; it < iterations; ++it) { + maximum[b] = maximum[b] > values[b][it] ? maximum[b] : values[b][it]; + } + } +#pragma unroll + for (int offset = lanes / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int b = 0; b < batch; ++b) { + const float other = __shfl_xor(maximum[b], offset, lanes); + maximum[b] = maximum[b] < other ? other : maximum[b]; + } + } +#pragma unroll + for (int b = 0; b < batch; ++b) { + sums[b] = 0.0f; +#pragma unroll + for (int it = 0; it < iterations; ++it) { + values[b][it] = std::exp(values[b][it] - maximum[b]); + sums[b] += values[b][it]; + } + } +#pragma unroll + for (int offset = lanes / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int b = 0; b < batch; ++b) { + sums[b] += __shfl_xor(sums[b], offset, lanes); + } + } +#pragma unroll + for (int b = 0; b < batch; ++b) { + if (first + b >= rows) { + break; + } +#pragma unroll + for (int it = 0; it < iterations; ++it) { + const int col = lane + it * lanes; + if (col < width) { + output[(first + b) * width + col] = values[b][it] / sums[b]; + } + } + } +} + +template +static __device__ __forceinline__ float combine(float a, float b) { + if constexpr (SUM) { + return a + b; + } else { + return a < b ? b : a; + } +} + +template +static __device__ __forceinline__ float block_reduce(float value, float * shared) { + const int tid = int(threadIdx.x), lane = tid % 32, warp = tid / 32; +#pragma unroll + for (int offset = 16; offset > 0; offset /= 2) { + value = combine(value, __shfl_down(value, offset, 32)); + } + __syncthreads(); + if (lane == 0) { + shared[warp] = value; + } + __syncthreads(); + value = tid < 16 ? shared[lane] : (SUM ? 0.0f : -std::numeric_limits::max()); + if (warp == 0) { +#pragma unroll + for (int offset = 16; offset > 0; offset /= 2) { + value = combine(value, __shfl_down(value, offset, 32)); + } + } + if (tid == 0) { + if constexpr (SUM) { + shared[0] = 1.0f / value; + } else { + shared[0] = value; + } + } + __syncthreads(); + return shared[0]; +} + +struct alignas(16) vector4 { float val[4]; }; + +template +static __device__ __forceinline__ float accumulate(float acc, float value, float maximum) { + if constexpr (SUM) { + return acc + __expf(value - maximum); + } else { + return ::max(acc, value); + } +} + +template +static __device__ __forceinline__ float local_reduce(const float * data, int size, float maximum) { + float result = SUM ? 0.0f : -std::numeric_limits::max(); + int offset = int(threadIdx.x); + if constexpr (DIVISIBLE) { + for (; offset * 4 < size; offset += 512) { + const vector4 values = reinterpret_cast(data)[offset]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + result = accumulate(result, values.val[j], maximum); + } + } + } else { + const int shift = int(uintptr_t(data) % 16) / int(sizeof(float)); + if (shift > 0) { + data -= shift; + size += shift; + if (offset >= shift && offset < size) { + result = accumulate(result, data[offset], maximum); + } + size -= size < 512 ? size : 512; + data += 512; + } + const int last = size % (4 * 512); + for (; offset * 4 < size - last; offset += 512) { + const vector4 values = reinterpret_cast(data)[offset]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + result = accumulate(result, values.val[j], maximum); + } + } + for (offset = size - last + int(threadIdx.x); offset < size; offset += 512) { + result = accumulate(result, data[offset], maximum); + } + } + return result; +} + +template +static __global__ void fast(const float * input, float * output, int width) { + extern __shared__ float shared[]; + input += int(blockIdx.x) * width; + output += int(blockIdx.x) * width; + const float maximum = block_reduce(local_reduce(input, width, 0.0f), shared); + const float inverse = block_reduce(local_reduce(input, width, maximum), shared); + if constexpr (DIVISIBLE) { + for (int offset = int(threadIdx.x); offset * 4 < width; offset += 512) { + const vector4 values = reinterpret_cast(input)[offset]; + vector4 result; +#pragma unroll + for (int j = 0; j < 4; ++j) { + result.val[j] = __expf(values.val[j] - maximum) * inverse; + } + reinterpret_cast(output)[offset] = result; + } + } else { + for (int col = int(threadIdx.x); col < width; col += 512) { + output[col] = __expf(input[col] - maximum) * inverse; + } + } +} + +template +static inline void launch_persistent(const float * input, float * output, int width, int rows, hipStream_t stream) { + constexpr int power = 1 << LOG2_WIDTH; + constexpr int lanes = power < 32 ? power : 32; + constexpr int batch = power <= 128 ? 2 : 1; + constexpr int groups = 128 / lanes; + persistent<<<(rows + groups * batch - 1) / (groups * batch), dim3(lanes, groups), 0, stream>>>( + input, output, width, rows); +} + +// Preconditions: F32, contiguous, aligned base, width in [16,4096], rows > 0, +// wave32, and total byte span <= INT_MAX. One launch, no allocation. +static inline void launch(const float * input, float * output, int width, int rows, hipStream_t stream) { + if (width > 2048) { + if (width % 4 == 0) { + fast<<>>(input, output, width); + } else { + fast<<>>(input, output, width); + } + return; + } + int log2_width = 4; + while ((1 << log2_width) < width) { + ++log2_width; + } + switch (log2_width) { +#define GGML_VISION_SOFTMAX_CASE(N) case N: launch_persistent(input, output, width, rows, stream); break + GGML_VISION_SOFTMAX_CASE(4); + GGML_VISION_SOFTMAX_CASE(5); + GGML_VISION_SOFTMAX_CASE(6); + GGML_VISION_SOFTMAX_CASE(7); + GGML_VISION_SOFTMAX_CASE(8); + GGML_VISION_SOFTMAX_CASE(9); + GGML_VISION_SOFTMAX_CASE(10); + GGML_VISION_SOFTMAX_CASE(11); +#undef GGML_VISION_SOFTMAX_CASE + } +} + +} // namespace ggml_vision_softmax + +/* +From PyTorch: + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +From Caffe2: + +Copyright (c) 2016-present, Facebook Inc. All rights reserved. + +All contributions by Facebook: +Copyright (c) 2016 Facebook Inc. + +All contributions by Google: +Copyright (c) 2015 Google Inc. +All rights reserved. + +All contributions by Yangqing Jia: +Copyright (c) 2015 Yangqing Jia +All rights reserved. + +All contributions by Kakao Brain: +Copyright 2019-2020 Kakao Brain + +All contributions by Cruise LLC: +Copyright (c) 2022 Cruise LLC. +All rights reserved. + +All contributions by Tri Dao: +Copyright (c) 2024 Tri Dao. +All rights reserved. + +All contributions by Arm: +Copyright (c) 2021, 2023-2025 Arm Limited and/or its affiliates + +All contributions from Caffe: +Copyright(c) 2013, 2014, 2015, the respective contributors +All rights reserved. + +All other contributions: +Copyright(c) 2015, 2016 the respective contributors +All rights reserved. + +Caffe2 uses a copyright model similar to Caffe: each contributor holds +copyright over their contributions to Caffe2. The project versioning records +all such contribution and copyright details. If a contributor wants to further +mark their specific copyright on a particular contribution, they should +indicate their copyright solely in the commit message of the change when it is +committed. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-softmax.cu b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-softmax.cu new file mode 100644 index 000000000..78747b946 --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-softmax.cu @@ -0,0 +1,52 @@ +#include "vision-softmax.cuh" + +#if defined(GGML_HIP_DS4V_VISION) +#include "vision-softmax-kernels.cuh" +#include + +bool ggml_hip_vision_softmax_f32_supported(int device, const ggml_tensor * op) { + const auto & info = ggml_cuda_info(); + if (device < 0 || device >= info.device_count || info.devices[device].warp_size != 32 || + !op || op->op != GGML_OP_SOFT_MAX_VISION_F32 || !op->src[0] || + op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || + op->ne[0] < 16 || op->ne[0] > 4096) { + return false; + } + int64_t elements = 1; + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + if (op->ne[i] != op->src[0]->ne[i] || op->ne[i] <= 0 || + op->ne[i] > (INT_MAX / int64_t(sizeof(float))) / elements) { + return false; + } + elements *= op->ne[i]; + } + if (!ggml_is_contiguous(op) || !ggml_is_contiguous(op->src[0]) || + op->view_offs % 16 != 0 || op->src[0]->view_offs % 16 != 0 || + uintptr_t(op->data) % 16 != 0 || uintptr_t(op->src[0]->data) % 16 != 0) { + return false; + } + for (int i = 1; i < GGML_MAX_SRC; ++i) { + if (op->src[i]) { + return false; + } + } + return true; +} + +void ggml_cuda_op_soft_max_vision_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { + GGML_ASSERT(ggml_hip_vision_softmax_f32_supported(ctx.device, dst)); + const int width = int(dst->ne[0]); + const int rows = int(ggml_nelements(dst) / width); + const auto * input = static_cast(dst->src[0]->data); + auto * output = static_cast(dst->data); + cudaStreamCaptureStatus capture; + CUDA_CHECK(cudaStreamIsCapturing(ctx.stream(), &capture)); + GGML_ASSERT(capture == cudaStreamCaptureStatusNone); + // A contiguous Torch allocation has an aligned base; row misalignment is + // reproduced inside the non-vector-width branch of the kernel. + GGML_ASSERT(uintptr_t(input) % 16 == 0 && uintptr_t(output) % 16 == 0); + ggml_vision_softmax::launch(input, output, width, rows, ctx.stream()); + CUDA_CHECK(cudaGetLastError()); + ++ctx.vision_softmax_launches; +} +#endif diff --git a/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-softmax.cuh b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-softmax.cuh new file mode 100644 index 000000000..9dcfb5ce9 --- /dev/null +++ b/server/deps/llama.cpp/ggml/src/ggml-cuda/vision-softmax.cuh @@ -0,0 +1,8 @@ +#pragma once + +#include "common.cuh" + +#if defined(GGML_HIP_DS4V_VISION) +bool ggml_hip_vision_softmax_f32_supported(int device, const ggml_tensor * op); +void ggml_cuda_op_soft_max_vision_f32(ggml_backend_cuda_context & ctx, ggml_tensor * dst); +#endif diff --git a/server/deps/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt b/server/deps/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt index 67dfedbae..0c20b89f0 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt +++ b/server/deps/llama.cpp/ggml/src/ggml-hip/CMakeLists.txt @@ -45,6 +45,7 @@ endif() find_package(hip REQUIRED) find_package(hipblas REQUIRED) +find_package(hipblaslt QUIET) find_package(rocblas REQUIRED) if (GGML_HIP_RCCL) @@ -152,6 +153,11 @@ else() set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE HIP) endif() +# Preserve source eager F32 softmax arithmetic; the kernel explicitly selects +# its fast exponential calls and must not gain reciprocal/FMA substitutions. +set_property(SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/../ggml-cuda/vision-softmax.cu" + APPEND PROPERTY COMPILE_OPTIONS -fno-fast-math -ffp-contract=off) + if (GGML_STATIC) message(FATAL_ERROR "Static linking not supported for HIP/ROCm") endif() @@ -168,3 +174,14 @@ target_link_directories(ggml-hip BEFORE PRIVATE "${GGML_HIP_RUNTIME_DIR}") target_link_options(ggml-hip PRIVATE "-L${GGML_HIP_RUNTIME_DIR}") target_link_libraries(ggml-hip PRIVATE ggml-base hip::host roc::rocblas roc::hipblas) + +# The DS4V vision tower ops use a fused hipBLASLt BF16 bias matmul. Without the +# hipBLASLt development package they are compiled out: the ops report +# unsupported and the server refuses --mmproj at startup. +if (hipblaslt_FOUND) + message(STATUS "hipBLASLt found: building the DS4V vision ops") + target_compile_definitions(ggml-hip PRIVATE GGML_HIP_DS4V_VISION) + target_link_libraries(ggml-hip PRIVATE roc::hipblaslt) +else() + message(STATUS "hipBLASLt not found: DS4V vision ops disabled") +endif() diff --git a/server/deps/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp b/server/deps/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp index 3c5990dae..1fb22b9b2 100644 --- a/server/deps/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/server/deps/llama.cpp/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -1984,6 +1984,9 @@ static ggml_backend_buffer_type_t ggml_backend_rpc_device_get_buffer_type(ggml_b } static bool ggml_backend_rpc_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + // This opt-in local HIP extension must not be sent to an older RPC peer. + if (op->op == GGML_OP_MUL_MAT_BIAS_BF16 || op->op == GGML_OP_RMS_NORM_VISION_F32 || + op->op == GGML_OP_SOFT_MAX_VISION_F32 || op->op == GGML_OP_MUL_MAT_VISION_AV_F32) return false; GGML_UNUSED(dev); GGML_UNUSED(op); //TODO: call the remote backend and cache the results diff --git a/server/deps/llama.cpp/ggml/src/ggml.c b/server/deps/llama.cpp/ggml/src/ggml.c index da4ef4405..b2c55aacd 100644 --- a/server/deps/llama.cpp/ggml/src/ggml.c +++ b/server/deps/llama.cpp/ggml/src/ggml.c @@ -1200,11 +1200,14 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "MUL_MAT_GROUPED_SRC", "PAGED_ATTN", - + "MUL_MAT_BIAS_BF16", + "RMS_NORM_VISION_F32", + "SOFT_MAX_VISION_F32", + "MUL_MAT_VISION_AV_F32", "DS4_MOE_COMBINE", }; -static_assert(GGML_OP_COUNT == 106, "GGML_OP_COUNT != 106"); +static_assert(GGML_OP_COUNT == 110, "GGML_OP_COUNT != 110"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1329,11 +1332,14 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "X*grouped(Y)", "paged_attn(q,k,v)", - + "bf16(X*Y+bias)", + "rms_norm_vision_f32(x)", + "soft_max_vision_f32(x)", + "vision_av_f32(v,p)", "ds4_moe_combine(down,w,shared)", }; -static_assert(GGML_OP_COUNT == 106, "GGML_OP_COUNT != 106"); +static_assert(GGML_OP_COUNT == 110, "GGML_OP_COUNT != 110"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -3312,6 +3318,62 @@ struct ggml_tensor * ggml_rms_norm_inplace( return ggml_rms_norm_impl(ctx, a, eps, true); } +// ggml_rms_norm_vision_f32 + +struct ggml_tensor * ggml_rms_norm_vision_f32( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps) { + GGML_ASSERT(a && a->type == GGML_TYPE_F32 && ggml_is_contiguous(a)); + GGML_ASSERT(a->ne[0] == 1024 && a->ne[1] >= 16 && a->ne[1] <= INT_MAX/1024); + GGML_ASSERT(a->ne[2] == 1 && a->ne[3] == 1); + GGML_ASSERT(isfinite(eps) && eps >= 0.0f); + struct ggml_tensor * result = ggml_dup_tensor(ctx, a); + ggml_set_op_params(result, &eps, sizeof(eps)); + result->op = GGML_OP_RMS_NORM_VISION_F32; + result->src[0] = a; + return result; +} + +// ggml_soft_max_vision_f32 + +struct ggml_tensor * ggml_soft_max_vision_f32( + struct ggml_context * ctx, + struct ggml_tensor * scores) { + GGML_ASSERT(scores && scores->type == GGML_TYPE_F32); + GGML_ASSERT(scores->ne[0] >= 16 && scores->ne[0] <= 4096); + int64_t elements = scores->ne[0]; + for (int i = 1; i < GGML_MAX_DIMS; ++i) { + GGML_ASSERT(scores->ne[i] > 0 && scores->ne[i] <= (INT_MAX/(int64_t)sizeof(float))/elements); + elements *= scores->ne[i]; + } + GGML_ASSERT(ggml_is_contiguous(scores)); + struct ggml_tensor * result = ggml_dup_tensor(ctx, scores); + result->op = GGML_OP_SOFT_MAX_VISION_F32; + result->src[0] = scores; + return result; +} + +// ggml_mul_mat_vision_av_f32 + +struct ggml_tensor * ggml_mul_mat_vision_av_f32( + struct ggml_context * ctx, + struct ggml_tensor * v, + struct ggml_tensor * probabilities) { + GGML_ASSERT(v && probabilities && v->type == GGML_TYPE_F32 && probabilities->type == GGML_TYPE_F32); + GGML_ASSERT(v->ne[0] == 64 && v->ne[1] == 16 && v->ne[3] == 1); + const int64_t n = v->ne[2]; + GGML_ASSERT(n >= 16 && n <= 4096); + GGML_ASSERT(probabilities->ne[0] == n && probabilities->ne[1] == n && + probabilities->ne[2] == 16 && probabilities->ne[3] == 1); + GGML_ASSERT(ggml_is_contiguous(v) && ggml_is_contiguous(probabilities)); + struct ggml_tensor * result = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 64, n, 16); + result->op = GGML_OP_MUL_MAT_VISION_AV_F32; + result->src[0] = v; + result->src[1] = probabilities; + return result; +} + // ggml_rms_norm_back struct ggml_tensor * ggml_rms_norm_back( @@ -3423,6 +3485,25 @@ struct ggml_tensor * ggml_mul_mat( return result; } +struct ggml_tensor * ggml_mul_mat_bias_bf16( + struct ggml_context * ctx, struct ggml_tensor * w, + struct ggml_tensor * x, struct ggml_tensor * b) { + GGML_ASSERT(w && x); + GGML_ASSERT(w->type == GGML_TYPE_BF16 && x->type == GGML_TYPE_BF16); + GGML_ASSERT(ggml_is_contiguous(w) && ggml_is_contiguous(x)); + GGML_ASSERT(w->ne[2] == 1 && w->ne[3] == 1 && x->ne[2] == 1 && x->ne[3] == 1); + GGML_ASSERT(w->ne[0] > 0 && w->ne[0] <= INT_MAX && w->ne[1] > 0 && w->ne[1] <= INT_MAX); + GGML_ASSERT(x->ne[0] == w->ne[0] && x->ne[1] > 0 && x->ne[1] <= INT_MAX); + if (b) { + GGML_ASSERT(b->type == GGML_TYPE_BF16 && ggml_is_contiguous(b)); + GGML_ASSERT(b->ne[0] == w->ne[1] && ggml_is_vector(b)); + } + struct ggml_tensor * result = ggml_new_tensor_2d(ctx, GGML_TYPE_BF16, w->ne[1], x->ne[1]); + result->op = GGML_OP_MUL_MAT_BIAS_BF16; + result->src[0] = w; result->src[1] = x; result->src[2] = b; + return result; +} + struct ggml_tensor * ggml_mul_mat_grouped_src( struct ggml_context * ctx, struct ggml_tensor * a, @@ -7992,6 +8073,10 @@ static void ggml_compute_backward( case GGML_OP_NONE: { // noop } break; + case GGML_OP_MUL_MAT_BIAS_BF16: // inference-only, no backward kernel + case GGML_OP_RMS_NORM_VISION_F32: + case GGML_OP_SOFT_MAX_VISION_F32: + case GGML_OP_MUL_MAT_VISION_AV_F32: case GGML_OP_COUNT: default: { GGML_ABORT("%s: unsupported ggml op for backward pass: %s\n", __func__, ggml_op_name(tensor->op)); diff --git a/server/docs/ENVIRONMENT.md b/server/docs/ENVIRONMENT.md index 3ccb2a7ec..7b30ed230 100644 --- a/server/docs/ENVIRONMENT.md +++ b/server/docs/ENVIRONMENT.md @@ -79,6 +79,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. | `LUCE_KVFLASH` | unset | Prefer the CLI: `--kvflash` (token count or `auto`). | | `LUCE_PREFIX_CACHE_SLOTS` | 32 | Container-entrypoint equivalent of `--prefix-cache-slots`; not read directly by the native binary. | | `LUCE_PREFILL_CACHE_SLOTS` | 0 | Container-entrypoint equivalent of `--prefill-cache-slots`; not read directly by the native binary. | +| `LUCE_MMPROJ` | unset | Container-entrypoint equivalent of `--mmproj` (vision projector path, enables image input); not read directly by the native binary. | | `LUCE_PREFILL_POOL_TRIM_TOKENS` | unset | OPT-IN: trim cached allocations from legacy CUDA/HIP device pools at completed Qwen3.5 prefill chunk boundaries after each configured token interval. Intended for long, shape-changing prefills on non-VMM devices; each trim synchronizes the target backend and retires captured graphs. | | `LUCE_SPLIT_FAST_ROLLBACK` | unset | OPT-IN: exact F32 checkpoints and replay-free rollback for local qwen35 target layer splits. Prefer `--target-split-fast-rollback`; adds checkpoint VRAM (~1.65 GiB for the measured Qwen3.6-27B q=16 split). | | `LUCE_STALL_TOOL_PREFIX` | unset | OPT-IN: recover a stalled tool call by injecting the prepared tool prefix when generation stops after an action suffix. | @@ -293,6 +294,7 @@ consolidation of this list into CLI flags is tracked as follow-up work. - `LUCE_NO_MOE_SWIGLU_FUSE` - qwen35moe_ffn.cpp - `LUCE_NO_PREAD` - deepseek4_loader.cpp - `LUCE_PROF` - prof_env.h +- `LUCE_MMPROJ` - scripts/entrypoint.sh (maps to `--mmproj`) - `LUCE_PREFILL_CACHE_SLOTS` - scripts/entrypoint.sh (maps to `--prefill-cache-slots`) - `LUCE_PREFILL_POOL_TRIM_TOKENS` - qwen35_backend.cpp (OPT-IN: trim legacy device pools during long prefills) - `LUCE_PREFILL_TIMING` - qwen35_backend.cpp (DEBUG: per-ubatch prefill build/alloc/compute timing) diff --git a/server/scripts/entrypoint.sh b/server/scripts/entrypoint.sh index dfcdcef91..9f8c2524d 100755 --- a/server/scripts/entrypoint.sh +++ b/server/scripts/entrypoint.sh @@ -501,6 +501,7 @@ CMD=("$LUCE_SERVER_BIN" "$LUCE_TARGET" [ -n "$DRAFT_ARG" ] && CMD+=(--ddtree --ddtree-budget "$LUCE_BUDGET") [ -n "$LUCE_DEFAULT_MAX_TOKENS" ] && CMD+=(--default-max-tokens "$LUCE_DEFAULT_MAX_TOKENS") [ -n "$LUCE_MODEL_NAME" ] && CMD+=(--model-name "$LUCE_MODEL_NAME") +[ -n "${LUCE_MMPROJ:-}" ] && CMD+=(--mmproj "$LUCE_MMPROJ") # `--lazy-draft` is silently dropped by the C++ server unless both # `--prefill-drafter` and `--draft` are present (look for the runtime # warning `--lazy-draft ignored: requires both --prefill-drafter and diff --git a/server/src/common/backend_args.h b/server/src/common/backend_args.h index 6b99132a9..8c2a295ff 100644 --- a/server/src/common/backend_args.h +++ b/server/src/common/backend_args.h @@ -54,6 +54,9 @@ struct BackendArgs { // Optional: speculative decode draft model (qwen35 only) std::optional draft_path; + // Optional: vision projector .gguf (deepseek4 only) + std::optional mmproj_path; + // Device placement DevicePlacement device; DevicePlacement draft_device; diff --git a/server/src/common/backend_factory.cpp b/server/src/common/backend_factory.cpp index 2f83a13ef..6e3dea2b5 100644 --- a/server/src/common/backend_factory.cpp +++ b/server/src/common/backend_factory.cpp @@ -190,6 +190,7 @@ std::unique_ptr construct_backend( Qwen35Config cfg; cfg.target_path = model.path; cfg.draft_path = speculation.draft_path; + cfg.mmproj_path = model.mmproj_path.value_or(""); cfg.device = placement.target; cfg.draft_gpu = placement.draft.gpu; cfg.remote_draft = placement.remote_draft; @@ -368,6 +369,7 @@ std::unique_ptr construct_backend( !placement.remote_target_shard.enabled()) { DeepSeek4BackendConfig cfg; cfg.model_path = model.path; + cfg.mmproj_path = model.mmproj_path.value_or(""); cfg.device = placement.target; cfg.stream_fd = execution.stream_fd; cfg.max_ctx = placement.target.max_ctx; diff --git a/server/src/common/backend_factory.h b/server/src/common/backend_factory.h index 95f99745b..11a53b809 100644 --- a/server/src/common/backend_factory.h +++ b/server/src/common/backend_factory.h @@ -36,6 +36,7 @@ class BackendPlan final { public: struct Model { std::string path; + std::optional mmproj_path; GgufModelInfo metadata; }; diff --git a/server/src/common/backend_plan.cpp b/server/src/common/backend_plan.cpp index 50fdff08d..adf3c7c9f 100644 --- a/server/src/common/backend_plan.cpp +++ b/server/src/common/backend_plan.cpp @@ -130,6 +130,7 @@ BackendPreparation BackendPlanBuilder::resolve( BackendPlan plan; plan.model_.path = std::move(args.model_path); + plan.model_.mmproj_path = std::move(args.mmproj_path); plan.model_.metadata = std::move(model); plan.placement_.target = std::move(args.device); diff --git a/server/src/common/copied_source_reclaim.h b/server/src/common/copied_source_reclaim.h new file mode 100644 index 000000000..16832eda2 --- /dev/null +++ b/server/src/common/copied_source_reclaim.h @@ -0,0 +1,64 @@ +#pragma once + +#include "moe_source_page_range.h" +#include +#include +#if defined(__linux__) +#include +#include +#include +#endif + +namespace luce::common { + +struct CopiedSourceAdviceResult { + size_t requested = 0; + int range_error = 0; + int madvise_error = 0; + int fadvise_error = 0; +}; + +// Contract: a read-only file mapping starting at file offset zero, its original +// borrowed fd, and a fully copied source tensor. Never advise anonymous/GPU +// allocations. The caller retains mapping ownership; future reads refault the +// original file bytes. Return values report advice acceptance, not bytes freed. +inline CopiedSourceAdviceResult reclaim_copied_file_source( + const void * mapping, size_t mapping_size, const void * source, + size_t source_size, int fd, const char * label, int layer = -1) { + CopiedSourceAdviceResult result; +#if defined(__linux__) + const long page_size = ::sysconf(_SC_PAGESIZE); + MoeSourcePageRange range; + const uintptr_t base = reinterpret_cast(mapping); + if (fd < 0 || page_size <= 0 || !moe_source_page_range( + base, mapping_size, reinterpret_cast(source), source_size, + static_cast(page_size), range)) { + result.range_error = EINVAL; + } else if (range.size != 0) { + const uintptr_t offset = range.address - base; + const auto max_offset = static_cast(std::numeric_limits::max()); + if (offset > max_offset || range.size > max_offset - offset) { + result.range_error = EOVERFLOW; + } else { + result.requested = range.size; + // Drop this mapping's PTE references before invalidating file cache. + if (::madvise(reinterpret_cast(range.address), range.size, MADV_DONTNEED) != 0) + result.madvise_error = errno; + // posix_fadvise returns the error number itself, not -1/errno. + result.fadvise_error = ::posix_fadvise(fd, static_cast(offset), + static_cast(range.size), POSIX_FADV_DONTNEED); + } + } + if (result.range_error || result.madvise_error || result.fadvise_error) { + std::fprintf(stderr, "[source-reclaim] %s layer=%d requested=%zu range_error=%d madvise_error=%d fadvise_error=%d\n", + label ? label : "?", layer, result.requested, result.range_error, + result.madvise_error, result.fadvise_error); + } +#else + (void) mapping; (void) mapping_size; (void) source; (void) source_size; + (void) fd; (void) label; (void) layer; +#endif + return result; +} + +} // namespace luce::common diff --git a/server/src/common/copied_source_upload.h b/server/src/common/copied_source_upload.h new file mode 100644 index 000000000..f6ccf3f98 --- /dev/null +++ b/server/src/common/copied_source_upload.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace luce::common { + +inline constexpr size_t COPIED_SOURCE_UPLOAD_CHUNK = 16 * 1024 * 1024; + +// Upload a verified file span through one reusable heap buffer. The synchronous +// callback must finish reading each chunk before returning. Its destination +// offset is relative to the tensor, never the file. No original file pointer +// reaches the callback. The caller retains and reuses scratch across tensors. +template +inline bool upload_copied_file_chunks( + const void * mapping, size_t mapping_size, size_t source_offset, + size_t source_size, std::vector & scratch, Upload && upload) { + const uintptr_t base = reinterpret_cast(mapping); + if (!mapping || mapping_size > std::numeric_limits::max() - base || + source_offset > mapping_size || source_size > mapping_size - source_offset) + return false; + if (source_size == 0) return true; + scratch.resize(std::min(source_size, COPIED_SOURCE_UPLOAD_CHUNK)); + const auto * source = static_cast(mapping) + source_offset; + for (size_t offset = 0; offset < source_size;) { + const size_t count = std::min(scratch.size(), source_size - offset); + std::memcpy(scratch.data(), source + offset, count); + upload(scratch.data(), offset, count); + offset += count; + } + return true; +} + +} // namespace luce::common diff --git a/server/src/common/feature_gate.cpp b/server/src/common/feature_gate.cpp index 2e32f9c52..fadbb6d40 100644 --- a/server/src/common/feature_gate.cpp +++ b/server/src/common/feature_gate.cpp @@ -47,6 +47,19 @@ std::string check_feature_compatibility( return "--target-shard-ipc-work-dir requires --target-shard-ipc-bin"; } + // ── vision projector × architecture / placement + if (args.mmproj_path.has_value()) { + if ((arch != "deepseek4" && arch != "qwen35") || args.device.is_layer_split() || + args.device.is_tensor_parallel() || args.remote_target_shard.enabled() || + args.max_concurrency != 1) { + return "--mmproj requires a local single-request DeepSeek4 or Qwen3.5 backend " + "that is not split across GPUs by layer or tensor"; + } + if (arch == "deepseek4" && target_backend != PlacementBackend::Hip) { + return "--mmproj with DeepSeek4 requires a HIP backend"; + } + } + // ── PFlash enablement × drafter model if (admission.pflash_enabled && !admission.pflash_drafter_configured) { diff --git a/server/src/common/generation_types.h b/server/src/common/generation_types.h index deefe911b..6c528f3e3 100644 --- a/server/src/common/generation_types.h +++ b/server/src/common/generation_types.h @@ -8,6 +8,7 @@ #include #include +#include "image_prompt.h" #include "sampler.h" namespace luce::common { @@ -26,6 +27,8 @@ struct BudgetHook { struct GenerateRequest { std::vector prompt; + // Backend-owned image payload bound to `prompt`; empty for text requests. + ImagePromptHandle images; int n_gen = 0; SamplerCfg sampler; bool do_sample = false; diff --git a/server/src/common/gpu_page_pool.cpp b/server/src/common/gpu_page_pool.cpp new file mode 100644 index 000000000..1189a0959 --- /dev/null +++ b/server/src/common/gpu_page_pool.cpp @@ -0,0 +1,93 @@ +#include "gpu_page_pool.h" + +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) +#include +#endif + +namespace luce::common { + +namespace { + +constexpr uint64_t OTHER_DRIVERS_MARGIN = 1ULL << 30; + +// Fields that together describe every page /proc/meminfo can attribute. +constexpr const char * ACCOUNTED[] = { + "MemFree", "Buffers", "Cached", "SwapCached", "AnonPages", "Slab", + "KernelStack", "PageTables", "SecPageTables", "Percpu", "Bounce", +}; + +#if defined(__linux__) +// False when no amdgpu device reports its GTT use, or when one has the counter +// but it cannot be read: the pool cannot then be told apart from pages that +// are in use. +bool live_gpu_host_bytes(uint64_t & total) { + total = 0; + bool found = false; + DIR * dir = opendir("/sys/class/drm"); + if (!dir) return false; + while (const dirent * entry = readdir(dir)) { + const std::string name = entry->d_name; + // card0, card1, ... but not the connector entries such as card0-DP-1. + if (name.rfind("card", 0) != 0 || name.find('-') != std::string::npos) continue; + std::ifstream used("/sys/class/drm/" + name + "/device/mem_info_gtt_used"); + if (!used.is_open()) continue; // not an amdgpu device + uint64_t bytes = 0; + if (!(used >> bytes)) { closedir(dir); return false; } + total += bytes; + found = true; + } + closedir(dir); + return found; +} +#endif + +} // namespace + +uint64_t reclaimable_gpu_page_pool_bytes(const char * meminfo_text, uint64_t live_gpu_host_bytes) { + if (!meminfo_text) return 0; + uint64_t total = 0, accounted = 0, huge_pages = 0, huge_page_kb = 0, hugetlb_kb = 0; + bool has_hugetlb = false; + std::istringstream lines(meminfo_text); + std::string line; + while (std::getline(lines, line)) { + const size_t colon = line.find(':'); + if (colon == std::string::npos) continue; + const std::string key = line.substr(0, colon); + const uint64_t value = std::strtoull(line.c_str() + colon + 1, nullptr, 10); + if (key == "MemTotal") total = value; + else if (key == "HugePages_Total") huge_pages = value; + else if (key == "Hugepagesize") huge_page_kb = value; + else if (key == "Hugetlb") { hugetlb_kb = value; has_hugetlb = true; } + else for (const char * field : ACCOUNTED) if (key == field) accounted += value; + } + // Hugetlb covers pools of every page size; older kernels only report the + // default pool's page count. + accounted += has_hugetlb ? hugetlb_kb : huge_pages * huge_page_kb; + if (total <= accounted) return 0; + const uint64_t unattributed = (total - accounted) * 1024; + const uint64_t in_use = live_gpu_host_bytes + OTHER_DRIVERS_MARGIN; + return unattributed > in_use ? unattributed - in_use : 0; +} + +uint64_t reclaimable_gpu_page_pool_bytes() { +#if defined(__linux__) + std::ifstream input("/proc/meminfo"); + if (!input) return 0; + uint64_t live = 0; + if (!live_gpu_host_bytes(live)) return 0; + std::stringstream text; + text << input.rdbuf(); + return reclaimable_gpu_page_pool_bytes(text.str().c_str(), live); +#else + return 0; +#endif +} + +} // namespace luce::common diff --git a/server/src/common/gpu_page_pool.h b/server/src/common/gpu_page_pool.h new file mode 100644 index 000000000..a6ad52385 --- /dev/null +++ b/server/src/common/gpu_page_pool.h @@ -0,0 +1,28 @@ +// Memory the GPU driver is holding for reuse. +// +// When a process frees GPU buffers that live in system RAM (GTT, and every +// allocation on an integrated GPU), the kernel's TTM layer keeps the pages in a +// pool instead of returning them. The next GPU allocation is served from that +// pool first, and the kernel shrinks it under memory pressure. /proc/meminfo +// does not count the pool in MemAvailable, so after one large run MemAvailable +// can read tens of GiB low even though that memory is there for the taking. +// +// The pool size itself is only visible to root (debugfs). This estimates it from +// what any process can read: RAM that /proc/meminfo attributes to nothing, +// minus the GPU buffers that are live right now (amdgpu sysfs). Measured on a +// Strix Halo + R9700 box it tracks the kernel counter within 0.8 GiB with the +// pool empty, full (61 GiB), and while 88 GiB of GPU buffers are live. +#pragma once + +#include + +namespace luce::common { + +// 0 when nothing can be read (non-Linux, no amdgpu). The result already has a +// 1 GiB margin taken off for pages owned by other drivers. +uint64_t reclaimable_gpu_page_pool_bytes(); + +// Same estimate from caller-supplied text, for tests. +uint64_t reclaimable_gpu_page_pool_bytes(const char * meminfo_text, uint64_t live_gpu_host_bytes); + +} // namespace luce::common diff --git a/server/src/common/image_prompt.h b/server/src/common/image_prompt.h new file mode 100644 index 000000000..2692b6f70 --- /dev/null +++ b/server/src/common/image_prompt.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include +#include + +namespace luce::common { + +struct EncodedImage { + std::string mime_type; + std::vector bytes; +}; + +class ImagePromptPayload { +public: + virtual ~ImagePromptPayload() = default; + virtual bool matches(const std::vector & tokens) const = 0; +}; + +using ImagePromptHandle = std::shared_ptr; + +} // namespace luce::common diff --git a/server/src/common/model_backend.h b/server/src/common/model_backend.h index 3937d006b..1f6e6afc4 100644 --- a/server/src/common/model_backend.h +++ b/server/src/common/model_backend.h @@ -24,6 +24,7 @@ #include "ggml-backend.h" #include "generation_types.h" #include "sampler.h" +#include "image_prompt.h" #include "concurrency/seq_engine.h" #include "placement/draft_residency.h" @@ -133,6 +134,27 @@ struct DaemonIO { struct ModelBackend { virtual ~ModelBackend() = default; + // Image input. A backend that supports it names the text its chat template + // turns into the image marker, and binds decoded images to a rendered prompt. + virtual bool supports_images() const { return false; } + virtual std::string image_placeholder() const { return {}; } + virtual bool prepare_images(std::vector & tokens, + std::vector images, + uint64_t context_capacity, + uint64_t output_reserve, + ImagePromptHandle & payload, + std::string & error) const { + (void) tokens; + (void) context_capacity; + (void) output_reserve; + if (!images.empty()) { + error = "this backend does not support image input"; + return false; + } + payload.reset(); + return true; + } + // Print the "[-daemon] ready ..." banner on stdout. virtual void print_ready_banner() const = 0; diff --git a/server/src/common/moe_hybrid_storage.cpp b/server/src/common/moe_hybrid_storage.cpp index dd7cd927f..7ee0ffaff 100644 --- a/server/src/common/moe_hybrid_storage.cpp +++ b/server/src/common/moe_hybrid_storage.cpp @@ -1,5 +1,7 @@ #include "moe_hybrid_storage.h" #include "moe_hybrid_types.h" +#include "moe_source_page_range.h" +#include "copied_source_reclaim.h" #include "ggml-cpu.h" #include "ggml-backend.h" @@ -9,6 +11,11 @@ #include #include #include +#include +#include +#if defined(__linux__) +#include +#endif #if defined(LUCE_BACKEND_CUDA) #include @@ -30,6 +37,12 @@ namespace luce::common { namespace { +void advise_copied_source(const void * mapping, size_t mapping_size, + const ExpertTensorFileData & tensor, int layer, int source_fd) { + reclaim_copied_file_source(mapping, mapping_size, tensor.data, tensor.size, + source_fd, "expert", layer); +} + void unregister_mix_tensor(ggml_tensor * tensor) { if (!tensor || !tensor->data) return; @@ -483,7 +496,10 @@ bool build_moe_hybrid_storage_from_file( std::string * err, int cache_slots, bool allocate_cold, - ggml_backend_t cold_gpu_backend) { + ggml_backend_t cold_gpu_backend, + const void * readonly_file_mmap, + size_t readonly_file_mmap_size, + int readonly_file_fd) { if (!placement.matches(cfg)) { if (err) *err = "placement does not match config"; @@ -691,6 +707,21 @@ bool build_moe_hybrid_storage_from_file( ggml_backend_tensor_set(dst.down_cold, slice_buf.data(), 0, slice_buf.size()); } } + // Only the mmap-retaining wrapper supplies proven file-mapping bounds. + // All synchronous hot/cold uploads have returned, and both temporary + // slice buffers are gone. Retain the mapping for future streaming reads. + if (readonly_file_mmap && readonly_file_fd >= 0 && moe_source_pageout_eligible( + out.cold_backend_kind == MoeHybridColdBackend::Gpu, + cfg.materialize_hot_experts, cfg.materialize_cold_experts, + allocate_cold && cold_count > 0 && dst.cold_buf != nullptr)) { + if (dst.fused_gate_up) { + advise_copied_source(readonly_file_mmap, readonly_file_mmap_size, fd.gate_up_exps, il, readonly_file_fd); + } else { + advise_copied_source(readonly_file_mmap, readonly_file_mmap_size, fd.gate_exps, il, readonly_file_fd); + advise_copied_source(readonly_file_mmap, readonly_file_mmap_size, fd.up_exps, il, readonly_file_fd); + } + advise_copied_source(readonly_file_mmap, readonly_file_mmap_size, fd.down_exps, il, readonly_file_fd); + } } return true; @@ -788,12 +819,13 @@ bool build_moe_hybrid_storage_from_file_with_mmap( MoeHybridStorage & out, std::string * err, int cache_slots, - ggml_backend_t cold_gpu_backend) { + ggml_backend_t cold_gpu_backend, + int readonly_file_fd) { // First build storage normally (hot GPU + cold CPU buffers). if (!build_moe_hybrid_storage_from_file( cfg, gpu_backend, placement, layer_descs, file_data, - out, err, cache_slots, true, cold_gpu_backend)) { + out, err, cache_slots, true, cold_gpu_backend, mmap_base, mmap_total_size, readonly_file_fd)) { return false; } diff --git a/server/src/common/moe_hybrid_storage.h b/server/src/common/moe_hybrid_storage.h index 2bccd39d6..e9f775a3e 100644 --- a/server/src/common/moe_hybrid_storage.h +++ b/server/src/common/moe_hybrid_storage.h @@ -264,6 +264,10 @@ int moe_hybrid_cache_swap_in(MoeHybridLayerStorage & st, int global_expert, ggml_backend_t gpu_backend); // Build hybrid storage by loading expert data directly from file (mmap). +// Optional: a caller opts in to advisory page-cache reclamation of completed +// materialized GPU layers by passing all three readonly_file_* arguments: the +// read-only mapping (which must start at file offset zero), its size, and the +// descriptor it was mapped from. Source pointers stay valid; later reads refault. bool build_moe_hybrid_storage_from_file( const MoeHybridConfig & cfg, ggml_backend_t gpu_backend, @@ -274,7 +278,10 @@ bool build_moe_hybrid_storage_from_file( std::string * err = nullptr, int cache_slots = 0, bool allocate_cold = true, - ggml_backend_t cold_gpu_backend = nullptr); + ggml_backend_t cold_gpu_backend = nullptr, + const void * readonly_file_mmap = nullptr, + size_t readonly_file_mmap_size = 0, + int readonly_file_fd = -1); // Spark: split a VRAM budget into a pinned-hot tier + an auto-sized expert // cache ring. target_bytes==0 keeps the current budget (use the card); @@ -289,6 +296,8 @@ MoeSparkBudget spark_budget_split(uint64_t expert_budget, uint64_t total_expert_ // mmap_base: pointer to start of mmap'd file. // mmap_total_size: total file size. // This variant populates out.layer_regions for use by MoeHybridStreamEngine. +// Optional fd is borrowed only during construction and must identify this +// offset-zero read-only mapping. The caller closes it after this call returns. bool build_moe_hybrid_storage_from_file_with_mmap( const MoeHybridConfig & cfg, ggml_backend_t gpu_backend, @@ -300,6 +309,7 @@ bool build_moe_hybrid_storage_from_file_with_mmap( MoeHybridStorage & out, std::string * err = nullptr, int cache_slots = 0, - ggml_backend_t cold_gpu_backend = nullptr); + ggml_backend_t cold_gpu_backend = nullptr, + int readonly_file_fd = -1); } // namespace luce::common diff --git a/server/src/common/moe_source_page_range.h b/server/src/common/moe_source_page_range.h new file mode 100644 index 000000000..ffb4b3564 --- /dev/null +++ b/server/src/common/moe_source_page_range.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +namespace luce::common { + +struct MoeSourcePageRange { + uintptr_t address = 0; + size_t size = 0; +}; + +// Select only complete pages inside BOTH a tensor and its verified read-only +// file mapping. An empty interior is valid; invalid/overflowing spans fail. +inline bool moe_source_page_range(uintptr_t mapping, size_t mapping_size, + uintptr_t tensor, size_t tensor_size, + size_t page_size, MoeSourcePageRange & out) { + out = {}; + const uintptr_t max = std::numeric_limits::max(); + if (!mapping || !page_size || (page_size & (page_size - 1)) != 0 || + mapping_size > max - mapping || tensor < mapping) return false; + const uintptr_t mapping_end = mapping + mapping_size; + if (tensor > mapping_end || tensor_size > mapping_end - tensor) return false; + const size_t padding = (page_size - tensor % page_size) % page_size; + if (padding > tensor_size) return true; + const size_t length = ((tensor_size - padding) / page_size) * page_size; + if (length) { + out.address = tensor + padding; + out.size = length; + } + return true; +} + +inline bool moe_source_pageout_eligible(bool cold_gpu, bool hot_materialized, + bool cold_materialized, bool cold_allocated) { + return cold_gpu && hot_materialized && cold_materialized && cold_allocated; +} + +} // namespace luce::common diff --git a/server/src/common/vision/image_decode.cpp b/server/src/common/vision/image_decode.cpp new file mode 100644 index 000000000..f9911290f --- /dev/null +++ b/server/src/common/vision/image_decode.cpp @@ -0,0 +1,372 @@ +#include "image_decode.h" + +#if !defined(LUCE_NO_IMAGE_CODECS) +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace luce::vision { +namespace { + +DecodeResult fail(DecodeError code, std::string message) { + DecodeResult result; + result.status = {code, std::move(message)}; + return result; +} + +#if !defined(LUCE_NO_IMAGE_CODECS) +DecodeStatus validate_encoded(const EncodedImageView & encoded, const DecodeLimits & limits) { + if (encoded.data == nullptr || encoded.size == 0) { + return {DecodeError::EmptyInput, "encoded image is empty"}; + } + if (limits.max_encoded_bytes == 0 || encoded.size > limits.max_encoded_bytes) { + return {DecodeError::EncodedTooLarge, "encoded image exceeds the configured byte limit"}; + } + return {}; +} + +DecodeStatus validate_decoded( + std::uint32_t width, + std::uint32_t height, + const DecodeLimits & limits, + std::size_t & output_bytes) { + if (width == 0 || height == 0) { + return {DecodeError::MalformedImage, "decoded image dimensions must be positive"}; + } + if (width > limits.max_dimension || height > limits.max_dimension) { + return {DecodeError::DecodedTooLarge, "decoded image dimension exceeds the limit"}; + } + const std::uint64_t pixels = static_cast(width) * height; + if (pixels > limits.max_decoded_pixels) { + return {DecodeError::DecodedTooLarge, "decoded image pixel count exceeds the limit"}; + } + if (pixels > std::numeric_limits::max() / 3) { + return {DecodeError::DecodedTooLarge, "decoded RGB byte count overflows size_t"}; + } + output_bytes = static_cast(pixels * 3); + return {}; +} + +bool checked_add_size(std::size_t left, std::size_t right, std::size_t & result) { + if (right > std::numeric_limits::max() - left) { + return false; + } + result = left + right; + return true; +} + +bool checked_mul_size(std::size_t left, std::size_t right, std::size_t & result) { + if (left != 0 && right > std::numeric_limits::max() / left) { + return false; + } + result = left * right; + return true; +} + +bool png_filtered_size( + std::uint32_t width, + std::uint32_t height, + std::uint32_t bits_per_pixel, + std::size_t & result) { + std::size_t whole_bytes = 0; + if (!checked_mul_size(width / 8, bits_per_pixel, whole_bytes)) { + return false; + } + const std::size_t partial_bytes = + ((width & 7U) * static_cast(bits_per_pixel) + 7) / 8; + std::size_t row_bytes = 0; + if (!checked_add_size(whole_bytes, partial_bytes, row_bytes) || + !checked_add_size(row_bytes, 1, row_bytes)) { + return false; + } + return checked_mul_size(row_bytes, height, result); +} + +bool png_idat_bound( + std::uint32_t width, + std::uint32_t height, + const LodePNGColorMode & color, + std::uint32_t interlace_method, + std::size_t & result) { + const std::uint32_t bits_per_pixel = lodepng_get_bpp(&color); + if (bits_per_pixel == 0) { + return false; + } + if (interlace_method == 0) { + return png_filtered_size(width, height, bits_per_pixel, result); + } + if (interlace_method != 1) { + return false; + } + + constexpr std::array x_start = {0, 4, 0, 2, 0, 1, 0}; + constexpr std::array y_start = {0, 0, 4, 0, 2, 0, 1}; + constexpr std::array x_step = {8, 8, 4, 4, 2, 2, 1}; + constexpr std::array y_step = {8, 8, 8, 4, 4, 2, 2}; + result = 0; + for (std::size_t pass = 0; pass < x_start.size(); ++pass) { + const std::uint32_t pass_width = width <= x_start[pass] + ? 0 + : (width - x_start[pass] + x_step[pass] - 1) / x_step[pass]; + const std::uint32_t pass_height = height <= y_start[pass] + ? 0 + : (height - y_start[pass] + y_step[pass] - 1) / y_step[pass]; + if (pass_width == 0 || pass_height == 0) { + continue; + } + std::size_t pass_size = 0; + if (!png_filtered_size(pass_width, pass_height, bits_per_pixel, pass_size) || + !checked_add_size(result, pass_size, result)) { + return false; + } + } + return result != 0; +} + +struct JpegErrorManager { + jpeg_error_mgr base; + std::jmp_buf jump; + char message[JMSG_LENGTH_MAX] = {}; + int warnings = 0; +}; + +extern "C" void jpeg_error_exit(j_common_ptr common) { + auto * error = reinterpret_cast(common->err); + error->base.format_message(common, error->message); + std::longjmp(error->jump, 1); +} + +extern "C" void jpeg_emit_message(j_common_ptr common, int level) { + auto * error = reinterpret_cast(common->err); + if (level < 0) { + ++error->warnings; + error->base.format_message(common, error->message); + } +} + +DecodeResult decode_jpeg(const EncodedImageView & encoded, const DecodeLimits & limits) { + DecodeResult result; + jpeg_decompress_struct decoder{}; + JpegErrorManager error{}; + volatile bool decoder_created = false; + std::uint8_t * volatile output = nullptr; + std::size_t output_bytes = 0; + + decoder.err = jpeg_std_error(&error.base); + error.base.error_exit = jpeg_error_exit; + error.base.emit_message = jpeg_emit_message; + if (setjmp(error.jump) != 0) { + if (decoder_created) { + jpeg_destroy_decompress(&decoder); + } + std::free(output); + return fail(DecodeError::MalformedImage, + error.message[0] == '\0' ? "malformed JPEG" : error.message); + } + + jpeg_create_decompress(&decoder); + decoder_created = true; + jpeg_mem_src(&decoder, encoded.data, static_cast(encoded.size)); + if (jpeg_read_header(&decoder, TRUE) != JPEG_HEADER_OK) { + jpeg_destroy_decompress(&decoder); + return fail(DecodeError::MalformedImage, "JPEG header is incomplete"); + } + if (decoder.jpeg_color_space == JCS_CMYK || decoder.jpeg_color_space == JCS_YCCK) { + jpeg_destroy_decompress(&decoder); + return fail(DecodeError::UnsupportedFormat, + "CMYK and YCCK JPEG images are not supported"); + } + decoder.out_color_space = JCS_RGB; + jpeg_calc_output_dimensions(&decoder); + if (decoder.output_width > std::numeric_limits::max() || + decoder.output_height > std::numeric_limits::max()) { + jpeg_destroy_decompress(&decoder); + return fail(DecodeError::DecodedTooLarge, "JPEG dimensions are out of range"); + } + if (const auto status = validate_decoded( + static_cast(decoder.output_width), + static_cast(decoder.output_height), + limits, + output_bytes); + !status) { + jpeg_destroy_decompress(&decoder); + result.status = status; + return result; + } + if (!jpeg_start_decompress(&decoder) || decoder.output_components != 3) { + jpeg_destroy_decompress(&decoder); + return fail(DecodeError::MalformedImage, "JPEG cannot be converted to RGB"); + } + + output = static_cast(std::malloc(output_bytes)); + if (output == nullptr) { + jpeg_destroy_decompress(&decoder); + return fail(DecodeError::AllocationFailed, "cannot allocate decoded JPEG RGB buffer"); + } + const std::size_t stride = static_cast(decoder.output_width) * 3; + while (decoder.output_scanline < decoder.output_height) { + JSAMPROW row = output + + static_cast(decoder.output_scanline) * stride; + if (jpeg_read_scanlines(&decoder, &row, 1) != 1) { + jpeg_destroy_decompress(&decoder); + std::free(output); + return fail(DecodeError::MalformedImage, "JPEG scanline data is incomplete"); + } + } + if (!jpeg_finish_decompress(&decoder) || error.warnings != 0) { + jpeg_destroy_decompress(&decoder); + std::free(output); + return fail(DecodeError::MalformedImage, + error.message[0] == '\0' ? "JPEG has decoding warnings" : error.message); + } + + result.image.width = static_cast(decoder.output_width); + result.image.height = static_cast(decoder.output_height); + jpeg_destroy_decompress(&decoder); + decoder_created = false; + try { + const auto * begin = output; + result.image.pixels.assign(begin, begin + output_bytes); + } catch (...) { + std::free(output); + return fail(DecodeError::AllocationFailed, "cannot own decoded JPEG RGB buffer"); + } + std::free(output); + result.status = {}; + return result; +} + +DecodeResult decode_png(const EncodedImageView & encoded, const DecodeLimits & limits) { + LodePNGState state; + lodepng_state_init(&state); + state.decoder.ignore_crc = 0; + state.decoder.ignore_critical = 0; + state.decoder.ignore_end = 0; + state.decoder.zlibsettings.ignore_adler32 = 0; + state.decoder.zlibsettings.ignore_nlen = 0; +#ifdef LODEPNG_COMPILE_ANCILLARY_CHUNKS + state.decoder.read_text_chunks = 0; + state.decoder.remember_unknown_chunks = 0; + state.decoder.max_icc_size = 16ULL * 1024ULL * 1024ULL; +#endif + unsigned width = 0; + unsigned height = 0; + const unsigned inspect_error = + lodepng_inspect(&width, &height, &state, encoded.data, encoded.size); + if (inspect_error != 0) { + lodepng_state_cleanup(&state); + return fail(DecodeError::MalformedImage, + std::string("malformed PNG: ") + lodepng_error_text(inspect_error)); + } + + std::size_t output_bytes = 0; + if (const auto status = validate_decoded(width, height, limits, output_bytes); !status) { + lodepng_state_cleanup(&state); + DecodeResult result; + result.status = status; + return result; + } + std::size_t idat_bound = 0; + if (!png_idat_bound( + width, + height, + state.info_png.color, + state.info_png.interlace_method, + idat_bound)) { + lodepng_state_cleanup(&state); + return fail(DecodeError::MalformedImage, "PNG filtered scanline size is invalid"); + } + state.decoder.zlibsettings.max_output_size = idat_bound; + + const bool grey16 = + state.info_png.color.colortype == LCT_GREY && state.info_png.color.bitdepth == 16; + state.info_raw.colortype = grey16 ? LCT_GREY : LCT_RGB; + state.info_raw.bitdepth = grey16 ? 16 : 8; + state.decoder.color_convert = grey16 ? 0 : 1; + unsigned char * output = nullptr; + unsigned decoded_width = 0; + unsigned decoded_height = 0; + const unsigned decode_error = lodepng_decode( + &output, &decoded_width, &decoded_height, &state, encoded.data, encoded.size); + lodepng_state_cleanup(&state); + if (decode_error != 0) { + std::free(output); + if (decode_error == 109) { + return fail(DecodeError::MalformedImage, + "PNG IDAT exceeds decoded geometry bound"); + } + if (decode_error == 83) { + return fail(DecodeError::AllocationFailed, "cannot allocate PNG decode buffer"); + } + return fail(DecodeError::MalformedImage, + std::string("malformed PNG: ") + lodepng_error_text(decode_error)); + } + if (decoded_width != width || decoded_height != height) { + std::free(output); + return fail(DecodeError::MalformedImage, "PNG dimensions changed during decode"); + } + + DecodeResult result; + result.image.width = width; + result.image.height = height; + try { + if (grey16) { + const std::size_t sample_count = static_cast(width) * height; + result.image.pixels.resize(output_bytes); + for (std::size_t sample = 0; sample < sample_count; ++sample) { + // Samples are big-endian; the high byte is the 8-bit value. + // (Clamping the 16-bit value to 255 turns the image white.) + const std::uint8_t channel = output[sample * 2]; + result.image.pixels[sample * 3] = channel; + result.image.pixels[sample * 3 + 1] = channel; + result.image.pixels[sample * 3 + 2] = channel; + } + } else { + result.image.pixels.assign(output, output + output_bytes); + } + } catch (...) { + std::free(output); + return fail(DecodeError::AllocationFailed, "cannot own decoded PNG RGB buffer"); + } + std::free(output); + result.status = {}; + return result; +} + +#endif // !LUCE_NO_IMAGE_CODECS +} // namespace + +DecodeResult decode_image(const EncodedImageView & encoded, const DecodeLimits & limits) { +#if defined(LUCE_NO_IMAGE_CODECS) + (void) encoded; (void) limits; + return fail(DecodeError::UnsupportedFormat, "this build has no image codecs (LUCE_IMAGE_CODECS=OFF)"); +#else + if (const auto status = validate_encoded(encoded, limits); !status) { + DecodeResult result; + result.status = status; + return result; + } + constexpr std::uint8_t png_signature[] = {137, 80, 78, 71, 13, 10, 26, 10}; + if (encoded.size >= sizeof(png_signature) && + std::memcmp(encoded.data, png_signature, sizeof(png_signature)) == 0) { + return decode_png(encoded, limits); + } + if (encoded.size >= 3 && encoded.data[0] == 0xFF && encoded.data[1] == 0xD8 && + encoded.data[2] == 0xFF) { + return decode_jpeg(encoded, limits); + } + return fail(DecodeError::UnsupportedFormat, "encoded image is neither JPEG nor PNG"); +#endif +} + +} // namespace luce::vision diff --git a/server/src/common/vision/image_decode.h b/server/src/common/vision/image_decode.h new file mode 100644 index 000000000..a31d81ce3 --- /dev/null +++ b/server/src/common/vision/image_decode.h @@ -0,0 +1,70 @@ +// JPEG and PNG decoding to 8-bit RGB. Model independent: every vision model +// starts from these pixels and does its own resizing and patching. +#pragma once + +#include +#include +#include +#include + +namespace luce::vision { + +struct EncodedImageView { + const std::uint8_t * data = nullptr; + std::size_t size = 0; +}; + +// Checked before any decoded buffer is allocated. +struct DecodeLimits { + std::size_t max_encoded_bytes = 16ULL * 1024ULL * 1024ULL; + std::uint64_t max_decoded_pixels = 64ULL * 1024ULL * 1024ULL; + std::uint32_t max_dimension = 65'535; +}; + +// Borrowed row-major RGB, three bytes per pixel. +struct DecodedRgbView { + std::uint32_t width = 0; + std::uint32_t height = 0; + const std::uint8_t * data = nullptr; + std::size_t size = 0; +}; + +struct DecodedRgb { + std::uint32_t width = 0; + std::uint32_t height = 0; + std::vector pixels; + + DecodedRgbView view() const { + return {width, height, pixels.data(), pixels.size()}; + } +}; + +enum class DecodeError { + None = 0, + EmptyInput, + EncodedTooLarge, + UnsupportedFormat, + MalformedImage, + DecodedTooLarge, + AllocationFailed, +}; + +struct DecodeStatus { + DecodeError code = DecodeError::None; + std::string message; + + explicit operator bool() const { return code == DecodeError::None; } +}; + +struct DecodeResult { + DecodeStatus status; + DecodedRgb image; + + explicit operator bool() const { return static_cast(status); } +}; + +DecodeResult decode_image( + const EncodedImageView & encoded, + const DecodeLimits & limits = {}); + +} // namespace luce::vision diff --git a/server/src/common/vision/image_resize.cpp b/server/src/common/vision/image_resize.cpp new file mode 100644 index 000000000..e58e089c7 --- /dev/null +++ b/server/src/common/vision/image_resize.cpp @@ -0,0 +1,257 @@ +#include "image_resize.h" + +#include +#include +#include +#include +#include +#include + +namespace luce::vision { +namespace { + +struct Status { + bool good = true; + std::string message; + explicit operator bool() const { return good; } +}; +// Pillow's fixed-point precision for 8-bit resampling. +constexpr int kPrecisionBits = 22; + +Status ok() { return {}; } +Status fail(std::string message) { return {false, std::move(message)}; } + +bool checked_mul(std::uint64_t a, std::uint64_t b, std::uint64_t & out) { + if (a != 0 && b > std::numeric_limits::max() / a) return false; + out = a * b; + return true; +} + +struct Coefficients { + int kernel_size = 0; + std::vector bounds; + std::vector weights; +}; + +// Pillow 12.3.0 Resample.c reference used for byte parity: +// https://github.com/python-pillow/Pillow/blob/12.3.0/src/libImaging/Resample.c +double bicubic(double x) { + constexpr double a = -0.5; + if (x < 0.0) { + x = -x; + } + if (x < 1.0) { + return ((a + 2.0) * x - (a + 3.0)) * x * x + 1.0; + } + if (x < 2.0) { + return (((x - 5.0) * x + 8.0) * x - 4.0) * a; + } + return 0.0; +} + +Status precompute_coefficients(int input_size, int output_size, Coefficients & out) { + if (input_size <= 0 || output_size <= 0) { + return fail("resample dimensions must be positive"); + } + const float input_begin = 0.0F; + const float input_end = static_cast(input_size); + double filter_scale = + (static_cast(input_end) - static_cast(input_begin)) / output_size; + const double scale = filter_scale; + if (filter_scale < 1.0) { + filter_scale = 1.0; + } + const double support = 2.0 * filter_scale; + const int kernel_size = static_cast(std::ceil(support)) * 2 + 1; + std::uint64_t coefficient_count = 0; + if (!checked_mul(static_cast(output_size), + static_cast(kernel_size), coefficient_count) || + coefficient_count > std::numeric_limits::max()) { + return fail("resample coefficient count overflow"); + } + + std::vector floating(static_cast(coefficient_count), 0.0); + out.bounds.resize(static_cast(output_size) * 2); + out.weights.resize(static_cast(coefficient_count)); + const double inverse_filter_scale = 1.0 / filter_scale; + for (int output = 0; output < output_size; ++output) { + const double center = input_begin + (output + 0.5) * scale; + int first = static_cast(center - support + 0.5); + if (first < 0) { + first = 0; + } + int count = static_cast(center + support + 0.5); + if (count > input_size) { + count = input_size; + } + count -= first; + double sum = 0.0; + const std::size_t base = static_cast(output) * kernel_size; + for (int index = 0; index < count; ++index) { + const double weight = bicubic( + (index + first - center + 0.5) * inverse_filter_scale); + floating[base + index] = weight; + sum += weight; + } + if (sum != 0.0) { + for (int index = 0; index < count; ++index) { + floating[base + index] /= sum; + } + } + out.bounds[static_cast(output) * 2] = first; + out.bounds[static_cast(output) * 2 + 1] = count; + } + + constexpr double scale_to_fixed = static_cast(1U << kPrecisionBits); + for (std::size_t index = 0; index < floating.size(); ++index) { + const double value = floating[index] * scale_to_fixed; + out.weights[index] = static_cast( + value < 0.0 ? value - 0.5 : value + 0.5); + } + out.kernel_size = kernel_size; + return ok(); +} + +std::uint8_t clip_fixed(std::int32_t value) { + const std::int32_t rounded = value >> kPrecisionBits; + return static_cast(std::clamp(rounded, 0, 255)); +} + +Status resize_horizontal( + const std::vector & input, + int input_width, + int input_height, + int output_width, + std::vector & output) { + Coefficients coeffs; + if (const auto status = precompute_coefficients(input_width, output_width, coeffs); !status) { + return status; + } + output.resize(static_cast(output_width) * input_height * 3); + for (int y = 0; y < input_height; ++y) { + for (int x = 0; x < output_width; ++x) { + const int first = coeffs.bounds[static_cast(x) * 2]; + const int count = coeffs.bounds[static_cast(x) * 2 + 1]; + const std::int32_t * weights = + coeffs.weights.data() + static_cast(x) * coeffs.kernel_size; + for (int channel = 0; channel < 3; ++channel) { + std::int32_t sum = 1 << (kPrecisionBits - 1); + for (int index = 0; index < count; ++index) { + const std::size_t source = + (static_cast(y) * input_width + first + index) * 3 + channel; + sum += static_cast(input[source]) * weights[index]; + } + const std::size_t destination = + (static_cast(y) * output_width + x) * 3 + channel; + output[destination] = clip_fixed(sum); + } + } + } + return ok(); +} + +Status resize_vertical( + const std::vector & input, + int input_width, + int input_height, + int output_height, + std::vector & output) { + Coefficients coeffs; + if (const auto status = precompute_coefficients(input_height, output_height, coeffs); !status) { + return status; + } + output.resize(static_cast(input_width) * output_height * 3); + for (int y = 0; y < output_height; ++y) { + const int first = coeffs.bounds[static_cast(y) * 2]; + const int count = coeffs.bounds[static_cast(y) * 2 + 1]; + const std::int32_t * weights = + coeffs.weights.data() + static_cast(y) * coeffs.kernel_size; + for (int x = 0; x < input_width; ++x) { + for (int channel = 0; channel < 3; ++channel) { + std::int32_t sum = 1 << (kPrecisionBits - 1); + for (int index = 0; index < count; ++index) { + const std::size_t source = + (static_cast(first + index) * input_width + x) * 3 + channel; + sum += static_cast(input[source]) * weights[index]; + } + const std::size_t destination = + (static_cast(y) * input_width + x) * 3 + channel; + output[destination] = clip_fixed(sum); + } + } + } + return ok(); +} + +Status resize_impl( + const std::vector & input, + int input_width, + int input_height, + int output_width, + int output_height, + std::vector & output) { + if (input_width == output_width && input_height == output_height) { + output = input; + return ok(); + } + + std::vector intermediate; + const bool vertical_first = + static_cast(input_height) > + static_cast(input_width) * 100 && + output_height < input_height; + if (vertical_first) { + if (const auto status = resize_vertical( + input, input_width, input_height, output_height, intermediate); + !status) { + return status; + } + if (output_width == input_width) { + output = std::move(intermediate); + return ok(); + } + return resize_horizontal( + intermediate, input_width, output_height, output_width, output); + } + + const std::vector * vertical_input = &input; + int vertical_width = input_width; + if (output_width != input_width) { + if (const auto status = resize_horizontal( + input, input_width, input_height, output_width, intermediate); + !status) { + return status; + } + vertical_input = &intermediate; + vertical_width = output_width; + } + if (output_height != input_height) { + return resize_vertical( + *vertical_input, vertical_width, input_height, output_height, output); + } + output = *vertical_input; + return ok(); +} + +} // namespace + +bool resize_rgb_bicubic(const std::vector & input, int input_width, int input_height, + int output_width, int output_height, + std::vector & output, std::string & error) { + error.clear(); + if (input_width <= 0 || input_height <= 0 || output_width <= 0 || output_height <= 0 || + input.size() != static_cast(input_width) * input_height * 3) { + error = "resize needs positive sizes and width*height*3 input bytes"; + return false; + } + try { + const Status status = resize_impl(input, input_width, input_height, output_width, output_height, output); + if (!status) error = status.message; + return bool(status); + } catch (const std::bad_alloc &) { + error = "resize allocation failed"; + return false; + } +} + +} // namespace luce::vision diff --git a/server/src/common/vision/image_resize.h b/server/src/common/vision/image_resize.h new file mode 100644 index 000000000..1ddad1ece --- /dev/null +++ b/server/src/common/vision/image_resize.h @@ -0,0 +1,23 @@ +// Bicubic resize of 8-bit RGB that reproduces Pillow's Image.resize(BICUBIC) +// byte for byte, including its antialiasing when shrinking. Vision models are +// trained on images resized this way, so every model's preprocessing uses it. +// One exception: an image more than 100 times taller than wide that is being +// shortened goes vertically first to keep the intermediate buffer small, where +// Pillow always goes horizontally first; the two orders can differ by a +// rounding step. +// Reference: Pillow 12.3.0 src/libImaging/Resample.c. +#pragma once + +#include +#include +#include + +namespace luce::vision { + +// `input` is row-major RGB, three bytes per pixel. Returns false and sets +// `error` on invalid sizes or allocation failure; `output` is then unspecified. +bool resize_rgb_bicubic(const std::vector & input, int input_width, int input_height, + int output_width, int output_height, + std::vector & output, std::string & error); + +} // namespace luce::vision diff --git a/server/src/common/vision/image_spans.h b/server/src/common/vision/image_spans.h new file mode 100644 index 000000000..bf1c14a82 --- /dev/null +++ b/server/src/common/vision/image_spans.h @@ -0,0 +1,75 @@ +// Where images sit inside a prompt. Model independent: a backend uses these to +// keep an image inside one prefill batch and to build its attention mask. +#pragma once + +#include +#include +#include +#include + +namespace luce::vision { + +// Half-open absolute token positions. `block` covers every token the image +// expanded to; `visible` is the part that attends bidirectionally. +struct TokenSpan { + std::uint64_t block_begin = 0; + std::uint64_t visible_begin = 0; + std::uint64_t visible_end = 0; + std::uint64_t block_end = 0; +}; + +// Borrowed, sorted by position, non-overlapping. +struct ImageSpanView { + const TokenSpan * data = nullptr; + size_t size = 0; +}; + +inline const TokenSpan * image_block_at(ImageSpanView spans, uint64_t position) { + for (size_t i = 0; i < spans.size; ++i) { + const auto & span = spans.data[i]; + if (position < span.block_begin) break; + if (position < span.block_end) return &span; + } + return nullptr; +} + +inline bool valid_image_spans(ImageSpanView spans, uint64_t prompt_size, + size_t max_images, uint64_t max_block_tokens) { + if (spans.size > max_images || (spans.size && !spans.data)) return false; + uint64_t previous_end = 0; + for (size_t i = 0; i < spans.size; ++i) { + const auto & span = spans.data[i]; + if (span.block_begin < previous_end || span.block_begin > span.visible_begin || + span.visible_begin >= span.visible_end || span.visible_end > span.block_end || + span.block_end > prompt_size || span.block_end - span.block_begin > max_block_tokens) { + return false; + } + previous_end = span.block_end; + } + return true; +} + +// A batch of about `proposed` tokens starting at `position` that does not cut +// an image in two: it stops before an image it cannot hold, and grows up to +// `capacity` to finish an image it starts with. Returns 0 when an image that +// starts here does not fit in `capacity`. +inline int atomic_image_chunk(ImageSpanView spans, uint64_t position, + int proposed, uint64_t remaining, int capacity) { + if (proposed <= 0 || capacity <= 0 || uint64_t(proposed) > remaining || + position > std::numeric_limits::max() - remaining) return 0; + uint64_t end = position + std::min(proposed, capacity); + for (size_t i = 0; i < spans.size; ++i) { + const auto & span = spans.data[i]; + if (span.block_end <= position) continue; + if (span.block_begin < position) return 0; + if (span.block_begin >= end) break; + if (end < span.block_end) { + end = span.block_begin == position ? span.block_end : span.block_begin; + break; + } + } + const uint64_t count = end - position; + return count && count <= remaining && count <= uint64_t(capacity) ? int(count) : 0; +} + +} // namespace luce::vision diff --git a/server/src/common/vision/mmproj_file.cpp b/server/src/common/vision/mmproj_file.cpp new file mode 100644 index 000000000..e85b32da0 --- /dev/null +++ b/server/src/common/vision/mmproj_file.cpp @@ -0,0 +1,108 @@ +#include "mmproj_file.h" + +#include "gguf.h" + +#include +#include + +namespace luce::vision { + +MmprojFile::~MmprojFile() { + if (buffer_) ggml_backend_buffer_free(buffer_); + if (ctx_) ggml_free(ctx_); + if (gguf_) gguf_free(gguf_); +} + +bool MmprojFile::load(const std::string & path, ggml_backend_t backend, std::string & error) { + error.clear(); + if (gguf_ || !backend) { error = "projector is already loaded or has no backend"; return false; } + + gguf_init_params params{}; + params.no_alloc = true; + params.ctx = &ctx_; + gguf_ = gguf_init_from_file(path.c_str(), params); + if (!gguf_ || !ctx_) { error = "cannot read projector GGUF: " + path; return false; } + + const int64_t arch = gguf_find_key(gguf_, "general.architecture"); + if (arch < 0 || gguf_get_kv_type(gguf_, arch) != GGUF_TYPE_STRING || + std::string(gguf_get_val_str(gguf_, arch)) != "clip") { + error = "not a clip projector file (general.architecture != clip)"; + return false; + } + + buffer_ = ggml_backend_alloc_ctx_tensors(ctx_, backend); + if (!buffer_) { error = "cannot allocate projector weights on the backend"; return false; } + ggml_backend_buffer_set_usage(buffer_, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + weight_bytes_ = ggml_backend_buffer_get_size(buffer_); + + std::unique_ptr file(std::fopen(path.c_str(), "rb"), std::fclose); + if (!file) { error = "cannot open projector file: " + path; return false; } + const size_t data_offset = gguf_get_data_offset(gguf_); + std::vector staging; + const int64_t n_tensors = gguf_get_n_tensors(gguf_); + for (int64_t i = 0; i < n_tensors; ++i) { + const char * name = gguf_get_tensor_name(gguf_, i); + ggml_tensor * t = ggml_get_tensor(ctx_, name); + if (!t) { error = std::string("projector tensor has no metadata: ") + name; return false; } + const size_t size = ggml_nbytes(t); + staging.resize(size); + const size_t offset = data_offset + gguf_get_tensor_offset(gguf_, i); +#if defined(_WIN32) + const int seek = _fseeki64(file.get(), (long long) offset, SEEK_SET); +#else + const int seek = fseeko(file.get(), (off_t) offset, SEEK_SET); +#endif + if (seek != 0 || std::fread(staging.data(), 1, size, file.get()) != size) { + error = std::string("projector file is truncated at tensor ") + name; + return false; + } + ggml_backend_tensor_set(t, staging.data(), 0, size); + } + return true; +} + +std::string MmprojFile::projector_type() const { + if (!gguf_) return {}; + const int64_t id = gguf_find_key(gguf_, "clip.projector_type"); + if (id < 0 || gguf_get_kv_type(gguf_, id) != GGUF_TYPE_STRING) return {}; + return gguf_get_val_str(gguf_, id); +} + +bool MmprojFile::u32(const char * key, uint32_t & out) const { + const int64_t id = gguf_ ? gguf_find_key(gguf_, key) : -1; + if (id < 0 || gguf_get_kv_type(gguf_, id) != GGUF_TYPE_UINT32) return false; + out = gguf_get_val_u32(gguf_, id); + return true; +} + +bool MmprojFile::f32(const char * key, float & out) const { + const int64_t id = gguf_ ? gguf_find_key(gguf_, key) : -1; + if (id < 0 || gguf_get_kv_type(gguf_, id) != GGUF_TYPE_FLOAT32) return false; + out = gguf_get_val_f32(gguf_, id); + return true; +} + +bool MmprojFile::f32_array(const char * key, std::vector & out) const { + const int64_t id = gguf_ ? gguf_find_key(gguf_, key) : -1; + if (id < 0 || gguf_get_kv_type(gguf_, id) != GGUF_TYPE_ARRAY || + gguf_get_arr_type(gguf_, id) != GGUF_TYPE_FLOAT32) return false; + const size_t count = gguf_get_arr_n(gguf_, id); + const auto * values = static_cast(gguf_get_arr_data(gguf_, id)); + out.clear(); + if (count > 0) out.assign(values, values + count); + return true; +} + +ggml_tensor * MmprojFile::tensor(const std::string & name) const { + return ctx_ ? ggml_get_tensor(ctx_, name.c_str()) : nullptr; +} + +bool MmprojFile::has_tensor_prefix(const std::string & prefix) const { + const int64_t n_tensors = gguf_ ? gguf_get_n_tensors(gguf_) : 0; + for (int64_t i = 0; i < n_tensors; ++i) { + if (std::string(gguf_get_tensor_name(gguf_, i)).rfind(prefix, 0) == 0) return true; + } + return false; +} + +} // namespace luce::vision diff --git a/server/src/common/vision/mmproj_file.h b/server/src/common/vision/mmproj_file.h new file mode 100644 index 000000000..f194ab05e --- /dev/null +++ b/server/src/common/vision/mmproj_file.h @@ -0,0 +1,51 @@ +// A multimodal projector file in llama.cpp's "clip" GGUF layout: the standard +// file published next to a vision-capable model. Holds the metadata and the +// tensors, loaded onto one backend. Each model's tower reads what it needs. +#pragma once + +#include "ggml-backend.h" +#include "ggml.h" + +#include +#include +#include +#include + +struct gguf_context; + +namespace luce::vision { + +class MmprojFile { +public: + MmprojFile() = default; + ~MmprojFile(); + MmprojFile(const MmprojFile &) = delete; + MmprojFile & operator=(const MmprojFile &) = delete; + + // Loads every tensor onto `backend`, which must outlive this object. + // One attempt per object: after a failure, discard it. + bool load(const std::string & path, ggml_backend_t backend, std::string & error); + + // "clip.projector_type", for example "qwen3vl_merger". Empty when absent. + std::string projector_type() const; + + // Metadata by full key. False when the key is absent or has another type. + bool u32(const char * key, uint32_t & out) const; + bool f32(const char * key, float & out) const; + bool f32_array(const char * key, std::vector & out) const; + + // nullptr when the file has no tensor of that name. + ggml_tensor * tensor(const std::string & name) const; + // True when any tensor name starts with `prefix`. + bool has_tensor_prefix(const std::string & prefix) const; + + size_t weight_bytes() const { return weight_bytes_; } + +private: + gguf_context * gguf_ = nullptr; + ggml_context * ctx_ = nullptr; + ggml_backend_buffer_t buffer_ = nullptr; + size_t weight_bytes_ = 0; +}; + +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_backend.cpp b/server/src/deepseek4/deepseek4_backend.cpp index 6d1784e5c..d45d9b762 100644 --- a/server/src/deepseek4/deepseek4_backend.cpp +++ b/server/src/deepseek4/deepseek4_backend.cpp @@ -4,6 +4,11 @@ #include "deepseek4_backend.h" #include "deepseek4_budget_hook.h" #include "deepseek4_internal.h" +#include "deepseek4_image_spans.h" +#include "deepseek4_image_budget.h" +#include "deepseek4_image_assembly.h" +#include "deepseek4_image_admission.h" +#include "../common/vision/image_decode.h" #include "luce.h" #include "deepseek4_snapshot.h" #include "deepseek4_page_layout.h" @@ -36,6 +41,47 @@ namespace luce::common { +class DeepSeek4ImagePrompt final : public ImagePromptPayload { +public: + bool matches(const std::vector & tokens) const override { + return tokens == prepared_.tokens; + } + vision::ImageSpanView spans() const { return {spans_.data(), spans_.size()}; } + +private: + friend class DeepSeek4Backend; + DeepSeek4ImagePrompt(const DeepSeek4Backend * owner, + vision::PreparedImagePrompt prepared, + std::vector encoded, std::shared_ptr lease) + : owner_(owner), prepared_(std::move(prepared)), encoded_(std::move(encoded)), + lease_(std::move(lease)) { + for (const auto & image : prepared_.images) spans_.push_back(image.layout.span); + } + + bool embed_chunk(const CpuEmbedder & embedder, size_t position, + int count, float * output) const { + if (!output || count <= 0 || embedder.n_embd <= 0) return false; + std::vector result; + std::string error; + const bool ok = vision::embed_image_prompt_chunk( + prepared_, materialized_, embedder.n_vocab, size_t(embedder.n_embd), + position, size_t(count), + [&](const int32_t * ids, size_t n, float * rows) { + return n <= size_t(std::numeric_limits::max()) && + embedder.embed(ids, int(n), rows); + }, result, error); + if (ok) std::copy(result.begin(), result.end(), output); + return ok; + } + + const DeepSeek4Backend * const owner_; + const vision::PreparedImagePrompt prepared_; + const std::vector encoded_; + const std::shared_ptr lease_; + std::vector spans_; + mutable std::vector> materialized_; +}; + namespace { using Clock = std::chrono::steady_clock; @@ -839,6 +885,7 @@ static bool compute_ds4_hybrid_budget_info(const DeepSeek4Weights & w, ggml_backend_t backend, uint64_t kv_bytes, bool all_cold, + bool with_vision, bool paged, Ds4HybridBudgetInfo & out, std::string * err) { @@ -867,9 +914,12 @@ static bool compute_ds4_hybrid_budget_info(const DeepSeek4Weights & w, // primary GPU's expert budget there. Paged serving owns its persistent // page tensors on the primary target. const uint64_t main_charge = all_cold && !paged ? 0 : out.kv_bytes; - if (out.gpu_total > out.core_bytes + main_charge + out.warm_bytes + out.safety_bytes) { - out.expert_budget = out.gpu_total - out.core_bytes - main_charge - out.warm_bytes - out.safety_bytes; - } + const uint64_t retained_workspace = with_vision && + (vision::detail::hip_bias_launches(backend) || vision::detail::hip_av_launches(backend)) + ? vision::detail::hip_bias_workspace(backend) : 0; + out.expert_budget = vision::remaining_expert_budget( + out.gpu_total, out.core_bytes, main_charge, out.warm_bytes, out.safety_bytes, + with_vision ? vision::SCRATCH_RESERVATION : 0, retained_workspace); if (out.expert_budget > out.mem.total_expert_bytes) { out.expert_budget = out.mem.total_expert_bytes; } @@ -991,6 +1041,245 @@ DeepSeek4Backend::~DeepSeek4Backend() { shutdown(); } +bool DeepSeek4Backend::prepare_images( + std::vector & tokens, std::vector images, + uint64_t context_capacity, uint64_t output_reserve, + ImagePromptHandle & payload, std::string & error) const { + if (images.empty()) { + // With a projector loaded, a marker left in a text prompt would be + // embedded as an ordinary token. Text-only backends skip the scan. + if (image_capable_) { + const int32_t marker = vision::ImageTokenizerContract{}.marker; + for (int32_t token : tokens) { + if (token == marker || token < 0 || token >= w_.n_vocab) { + error = "unbound image marker or invalid token in rendered prompt"; + return false; + } + } + } + payload.reset(); + return true; + } + if (!image_capable_) { + error = "image input requires a validated --mmproj projector and heterogeneous HIP sparse prefill"; + return false; + } + try { + if (images.size() > 4) { + error = "too many images in request"; + return false; + } + auto lease = image_request_gate_.try_acquire(); + if (!lease) { + error = "an image request is already in progress; retry after it completes"; + return false; + } + if (!vision::check_deepseek4_image_host_preparation(4ULL * 1024 * 1024 * 1024, error)) { + return false; + } + std::vector patches; + patches.reserve(images.size()); + size_t encoded_bytes = 0; + for (const auto & image : images) { + if (image.bytes.size() > 16ULL * 1024 * 1024 || + encoded_bytes > 32ULL * 1024 * 1024 - image.bytes.size()) { + error = "images exceed request byte limit"; + return false; + } + encoded_bytes += image.bytes.size(); + auto decoded = vision::decode_image({image.bytes.data(), image.bytes.size()}); + if (!decoded) { error = decoded.status.message; return false; } + auto processed = vision::preprocess_rgb(decoded.image.view(), 0); + if (!processed) { error = processed.status.message; return false; } + patches.push_back({processed.image.plan, std::move(processed.image.patches_bf16)}); + } + vision::ImagePromptLimits limits; + limits.context_capacity = context_capacity; + limits.output_reserve = output_reserve; + limits.max_expanded_tokens = std::min(context_capacity, vision::MAX_PREPARED_PROMPT_TOKENS); + auto prepared = vision::prepare_image_prompt(tokens, patches, limits); + if (!prepared) { error = prepared.message; return false; } + auto binding = std::shared_ptr( + new DeepSeek4ImagePrompt(this, std::move(prepared), std::move(images), std::move(lease))); + if (!vision::valid_image_spans(binding->spans(), binding->prepared_.tokens.size())) { + error = "invalid prepared image spans"; + return false; + } + std::vector expanded = binding->prepared_.tokens; + tokens.swap(expanded); + payload = std::move(binding); + return true; + } catch (const std::bad_alloc &) { + error = "image preparation allocation failed"; + return false; + } +} + +bool DeepSeek4Backend::materialize_images(const DeepSeek4ImagePrompt & images, + const DaemonIO & io, std::string & error) { + if (io.is_cancelled()) return false; + if (images.owner_ != this || !vision_ || parked_) { + error = "image binding does not belong to the loaded backend"; + return false; + } + ggml_backend_synchronize(backend_); + if (expert_backend_) ggml_backend_synchronize(expert_backend_); + if (spec_backend_) ggml_backend_synchronize(spec_backend_); + deepseek4_release_image_scratch(cache_, moe_hybrid_.get()); + // With the whole model on one GPU, text prefill keeps per-layer graph + // arenas alive between requests. They are rebuilt on demand, and the + // headroom measured below should not have to fit around them. + if (!moe_hybrid_) deepseek4_release_runtime_graphs(w_); + reset_deepseek4_dspark_runtime_cache(); + // Gallocr teardown leaves operator temporaries in legacy CUDA/HIP pools. + // Retire their captured executables and activation memos through the + // backend API before measuring headroom for the next image request. + const auto trim_pool = [](ggml_backend_t owner, const char * name) { + const size_t released = ggml_backend_cuda_trim_pool(owner); + std::fprintf(stderr, + "[deepseek4] image transition pool trim: owner=%s released=%zu bytes\n", + name, released); + }; + trim_pool(backend_, "primary"); + if (expert_backend_ && expert_backend_ != backend_) { + trim_pool(expert_backend_, "expert"); + } + if (spec_backend_ && spec_backend_ != backend_ && spec_backend_ != expert_backend_) { + trim_pool(spec_backend_, "spec"); + } + auto reserves = image_reserves_; + const uint64_t resident_workspace = + (vision::detail::hip_bias_launches(backend_) || vision::detail::hip_av_launches(backend_)) + ? vision::detail::hip_bias_workspace(backend_) : 0; + if (resident_workspace > vision::SCRATCH_RESERVATION) { + error = "resident vision workspace exceeds its reservation"; + return false; + } + // Current free memory already reflects weights, KV, optional drafter, + // snapshots and retained backend pools. Charge only upcoming work here. + reserves.primary_future_bytes = vision::SCRATCH_RESERVATION - resident_workspace + + 128ULL * 1024 * 1024; + if (!moe_hybrid_) { + uint64_t free_bytes = 0; + if (!vision::check_deepseek4_image_single_gpu_admission(backend_, reserves.primary_domain, + reserves.primary_future_bytes, free_bytes, error)) { + std::fprintf(stderr, "[deepseek4] image runtime admission failed (one GPU): required/free=%.3f/%.3f GiB: %s\n", + gib(reserves.primary_future_bytes), gib(free_bytes), error.c_str()); + return false; + } + } + vision::ImageAdmissionReport report; + MoeHybridConfig runtime_cfg = make_ds4_parent_worker_cfg(w_); + runtime_cfg.materialize_cold_experts = true; + runtime_cfg.cold_expert_backend = MoeHybridColdBackend::Gpu; + if (moe_hybrid_ && !vision::check_deepseek4_image_runtime_admission(runtime_cfg, + backend_, expert_backend_, reserves, report, error)) { + std::fprintf(stderr, + "[deepseek4] image runtime admission failed: primary required/free=%.3f/%.3f GiB " + "cold required/free=%.3f/%.3f GiB host required/available=%.3f/%.3f GiB: %s\n", + gib(report.primary_required_bytes), gib(report.primary_free_bytes), + gib(report.cold_required_bytes), gib(report.cold_free_bytes), + gib(report.host_required_bytes), gib(report.host_available_bytes), error.c_str()); + return false; + } + if (!images.materialized_.empty()) return true; + struct ReleaseScratch { + vision::VisionRuntime & runtime; + ~ReleaseScratch() { runtime.release_scratch(); } + } release{*vision_}; + try { + vision::ImageSentinels sentinels; + if (!vision_->sentinel(vision::Sentinel::Start, sentinels.start, error) || + !vision_->sentinel(vision::Sentinel::Pad, sentinels.pad, error) || + !vision_->sentinel(vision::Sentinel::Newline, sentinels.newline, error) || + !vision_->sentinel(vision::Sentinel::End, sentinels.end, error)) return false; + return vision::materialize_image_rows( + images.prepared_.images, sentinels, size_t(w_.n_embd), + [&](const vision::PromptImage & image, vision::ImageRaster & raster, + std::string & encode_error) { + std::vector patches(image.input.patches_bf16.size()); + for (size_t i = 0; i < patches.size(); ++i) { + const uint32_t bits = uint32_t(image.input.patches_bf16[i]) << 16; + std::memcpy(&patches[i], &bits, sizeof(bits)); + } + vision::VisionOutput output; + if (!vision_->encode(patches, + {int(image.input.plan.vit_rows), int(image.input.plan.vit_cols)}, + output, encode_error)) return false; + if (output.rows <= 0 || output.columns != w_.n_embd) { + encode_error = "vision output shape differs from decoder dimensions"; + return false; + } + raster = {size_t(output.rows), size_t(output.columns), std::move(output.embeddings)}; + return true; + }, [&] { return io.is_cancelled(); }, images.materialized_, error); + } catch (const std::bad_alloc &) { + error = "image materialization allocation failed"; + return false; + } +} + +// The whole model is already on one GPU: load the projector next to it and +// check that the image scratch still fits. +bool DeepSeek4Backend::init_single_gpu_vision() { + if (cfg_.mmproj_path.empty()) return true; + if (!load_vision()) return false; +#if defined(LUCE_BACKEND_HIP) || defined(GGML_USE_HIP) + hipDeviceProp_t properties{}; + if (hipGetDeviceProperties(&properties, cfg_.device.gpu) != hipSuccess) { + std::fprintf(stderr, "[deepseek4] cannot classify the image owner's memory domain\n"); + return false; + } + vision::ImageAdmissionReserves reserves; + reserves.primary_domain = properties.integrated || std::getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") + ? vision::ImageMemoryDomain::HostShared : vision::ImageMemoryDomain::Dedicated; + reserves.primary_future_bytes = estimate_ds4_cache_bytes(w_, cfg_.max_ctx > 0 ? cfg_.max_ctx : 8192) + + vision::SCRATCH_RESERVATION + 256ULL * 1024 * 1024; + uint64_t free_bytes = 0; + std::string error; + const bool admitted = vision::check_deepseek4_image_single_gpu_admission( + backend_, reserves.primary_domain, reserves.primary_future_bytes, free_bytes, error); + std::fprintf(stderr, "[deepseek4] image memory admission (one GPU): required/free=%.3f/%.3f GiB result=%s\n", + gib(reserves.primary_future_bytes), gib(free_bytes), admitted ? "admitted" : error.c_str()); + if (!admitted) return false; + image_reserves_ = reserves; + return true; +#else + return false; +#endif +} + +bool DeepSeek4Backend::load_vision() { + if (cfg_.mmproj_path.empty()) return true; + // The projector checks the decoder's width and vocabulary when it loads. + // Here: every layer carries a finite F32[n_expert] image router bias. + std::vector values(size_t(w_.n_expert)); + for (const auto & layer : w_.layers) { + const auto bias = layer.ffn_gate_bias_vl; + if (!bias || bias->type != GGML_TYPE_F32 || bias->ne[0] != w_.n_expert || + ggml_nelements(bias) != w_.n_expert) { + std::fprintf(stderr, "[deepseek4] --mmproj requires one F32[n_expert] image router bias per layer\n"); + return false; + } + ggml_backend_tensor_get(bias, values.data(), 0, values.size() * sizeof(float)); + if (!std::all_of(values.begin(), values.end(), [](float v) { return std::isfinite(v); })) { + std::fprintf(stderr, "[deepseek4] nonfinite image router bias\n"); + return false; + } + } + auto runtime = std::make_unique(); + std::string error; + if (!runtime->load(cfg_.mmproj_path, backend_, w_.n_embd, w_.n_vocab, error)) { + std::fprintf(stderr, "[deepseek4] projector load failed: %s\n", error.c_str()); + return false; + } + std::fprintf(stderr, + "[deepseek4] vision weights=%.3f GiB scratch reservation=%.3f GiB before expert placement\n", + gib(runtime->weight_bytes()), gib(vision::SCRATCH_RESERVATION)); + vision_ = std::move(runtime); + return true; +} + bool DeepSeek4Backend::requires_monolithic_model() const { return cfg_.paged_attention || cfg_.fused_decode || cfg_.fused_verify_f16_kv || @@ -1034,6 +1323,27 @@ bool DeepSeek4Backend::load_model() { // deployments outside the qualified R9700 + Strix Halo topology. const bool force_full = env_flag_enabled("LUCE_DS4_FORCE_FULL_LOAD"); const bool heterogeneous_tp = env_flag_enabled("LUCE_DS4_MOE_TP"); + if (!cfg_.mmproj_path.empty()) { + // Images run on one HIP GPU holding the whole model, or on two HIP GPUs + // that split the experts in process. Both need batched sparse prefill. + const auto tp = ds4_moe_tp_config(cfg_.device.gpu); + const bool two_gpu_ok = tp.in_process && tp.backend_valid && + tp.secondary_backend == PlacementBackend::Hip && + tp.secondary_gpu != cfg_.device.gpu && !tp.all_on_secondary && !force_full; + if (target_backend != PlacementBackend::Hip || cfg_.device.is_layer_split() || + cfg_.prefill_mode != PrefillAttentionMode::Sparse || + (tp.requested && !two_gpu_ok) || + env_flag_enabled("LUCE_DS4_DENSE_TP_MASK")) { + std::fprintf(stderr, "[deepseek4] --mmproj requires a HIP target with --ds4-prefill sparse, " + "on one GPU or with in-process expert owners on two distinct GPUs\n"); + return false; + } + if (!vision::detail::hip_bias_workspace(backend_)) { + std::fprintf(stderr, "[deepseek4] --mmproj needs the DS4V vision ops, which this build lacks " + "(hipBLASLt was not found when ggml-hip was configured)\n"); + return false; + } + } const bool need_monolithic = requires_monolithic_model() && !heterogeneous_tp; if (target_backend == PlacementBackend::Hip && @@ -1047,7 +1357,9 @@ bool DeepSeek4Backend::load_model() { cfg_.fused_decode ? "on" : "off", cfg_.fused_verify_f16_kv ? "on" : "off", prefill_attention_mode_name(cfg_.prefill_mode)); - if (!load_deepseek4_gguf(cfg_.model_path, backend_, w_)) { + TargetLoadPlan full_plan; + full_plan.load_ds4_image_bias = !cfg_.mmproj_path.empty(); + if (!load_deepseek4_gguf_partial(cfg_.model_path, backend_, full_plan, w_)) { if (prefill_attention_mode_is_approximate(cfg_.prefill_mode)) { std::fprintf(stderr, "[deepseek4] monolithic HIP load required for %s prefill\n", @@ -1059,6 +1371,7 @@ bool DeepSeek4Backend::load_model() { cfg_.model_path.c_str()); return false; } + if (!init_single_gpu_vision()) return false; } else if (target_backend == PlacementBackend::Hip || heterogeneous_tp) { std::fprintf(stderr, "[deepseek4] heterogeneous target detected; using hybrid expert load path\n"); @@ -1482,6 +1795,7 @@ bool DeepSeek4Backend::init() { std::fprintf(stderr, "[deepseek4] LUCE_DS4_SPEC set but LUCE_DS4_DRAFT gguf missing\n"); } } + image_capable_ = vision_ != nullptr; return true; } @@ -1576,7 +1890,7 @@ bool DeepSeek4Backend::compute_uniform_hybrid_placement(const DeepSeek4Weights & const Ds4MoeTpConfig tp = ds4_moe_tp_config(cfg_.device.gpu); if (!compute_ds4_hybrid_budget_info( w, backend_, kv_bytes, tp.all_on_secondary, - cfg_.paged_attention, budget, err)) { + vision_ != nullptr, cfg_.paged_attention, budget, err)) { return false; } @@ -1795,12 +2109,15 @@ bool DeepSeek4Backend::compute_uniform_hybrid_placement(const DeepSeek4Weights & bool DeepSeek4Backend::init_hybrid_model() { TargetLoadPlan plan; plan.skip_expert_tensors = true; + plan.load_ds4_image_bias = !cfg_.mmproj_path.empty(); if (!load_deepseek4_gguf_partial(cfg_.model_path, backend_, plan, w_)) { - std::fprintf(stderr, "[deepseek4] failed to partially load model for hybrid mode: %s\n", - cfg_.model_path.c_str()); + std::fprintf(stderr, "[deepseek4] failed to partially load model for hybrid mode: %s (%s)\n", + cfg_.model_path.c_str(), luce_last_error()); return false; } + if (!load_vision()) return false; + std::string err; const int max_ctx = cfg_.max_ctx > 0 ? cfg_.max_ctx : 8192; if (!compute_uniform_hybrid_placement( @@ -1810,6 +2127,11 @@ bool DeepSeek4Backend::init_hybrid_model() { } if (moe_placement_.total_hot >= w_.n_layer * w_.n_expert) { + if (vision_) { + std::fprintf(stderr, "[deepseek4] image prefill requires resident cold experts on the secondary HIP device\n"); + return false; + } + vision_.reset(); free_deepseek4_weights(w_); if (!load_deepseek4_gguf(cfg_.model_path, backend_, w_)) { std::fprintf(stderr, "[deepseek4] failed to reload full model after placement: %s\n", @@ -1857,6 +2179,7 @@ bool DeepSeek4Backend::init_hybrid_model() { std::fprintf(stderr, "[deepseek4] %s experts cannot decode from hybrid/cold " "placement; falling back to monolithic full load\n", m.what); + vision_.reset(); free_deepseek4_weights(w_); if (!load_deepseek4_gguf(cfg_.model_path, backend_, w_)) { std::fprintf(stderr, @@ -1930,6 +2253,55 @@ bool DeepSeek4Backend::init_hybrid_model() { hybrid_cfg.materialize_cold_experts = true; hybrid_cfg.cold_expert_backend = MoeHybridColdBackend::Gpu; } + if (vision_) { +#if defined(LUCE_BACKEND_HIP) || defined(GGML_USE_HIP) + hipDeviceProp_t primary_properties{}, cold_properties{}; + if (hipGetDeviceProperties(&primary_properties, cfg_.device.gpu) != hipSuccess || + hipGetDeviceProperties(&cold_properties, tp.secondary_gpu) != hipSuccess) { + std::fprintf(stderr, "[deepseek4] cannot classify image owner memory domains\n"); + return fail_hybrid_init(); + } + const bool unified = std::getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr; + const uint64_t resident_workspace = + (vision::detail::hip_bias_launches(backend_) || vision::detail::hip_av_launches(backend_)) + ? vision::detail::hip_bias_workspace(backend_) : 0; + if (resident_workspace > vision::SCRATCH_RESERVATION) return fail_hybrid_init(); + constexpr uint64_t mib = 1024ULL * 1024; + vision::ImageAdmissionReserves reserves; + reserves.primary_domain = primary_properties.integrated || unified + ? vision::ImageMemoryDomain::HostShared : vision::ImageMemoryDomain::Dedicated; + reserves.cold_domain = cold_properties.integrated || unified + ? vision::ImageMemoryDomain::HostShared : vision::ImageMemoryDomain::Dedicated; + reserves.duplicate_hot_on_cold = env_flag_enabled("LUCE_MOE_DUPLICATE_HOT_ON_COLD"); + reserves.primary_future_bytes = estimate_ds4_cache_bytes(w_, max_ctx) + + vision::SCRATCH_RESERVATION - resident_workspace + (256 + 512) * mib; + reserves.cold_future_bytes = 512 * mib; + reserves.cold_runtime_reservation_bytes = 2048 * mib; + reserves.max_chunk_tokens = 1024; + // The image gate retains one request through preprocessing and serving. + // These are named headroom reservations; admission also accounts for + // exact selected storage, copy growth, and shared physical host RAM. + reserves.host_request_bytes = 4096 * mib; + reserves.host_loader_overhead_bytes = 1024 * mib; + reserves.host_runtime_bytes = 512 * mib; + vision::ImageAdmissionReport report; + const bool admitted = vision::check_deepseek4_image_admission( + w_, moe_placement_, hybrid_cfg, backend_, expert_backend_, reserves, report, err); + std::fprintf(stderr, + "[deepseek4] image memory admission: primary required/free=%.3f/%.3f GiB " + "cold required/free=%.3f/%.3f GiB host+UMA required/available=%.3f/%.3f GiB " + "cold activation estimate/reservation=%.3f/%.3f GiB result=%s\n", + gib(report.primary_required_bytes), gib(report.primary_free_bytes), + gib(report.cold_required_bytes), gib(report.cold_free_bytes), + gib(report.host_required_bytes), gib(report.host_available_bytes), + gib(report.cold_activation_estimate_bytes), gib(report.cold_runtime_reservation_bytes), + admitted ? "admitted" : err.c_str()); + if (!admitted) return fail_hybrid_init(); + image_reserves_ = reserves; +#else + return fail_hybrid_init(); +#endif + } if (!build_deepseek4_moe_hybrid_storage_from_file_with_mmap( cfg_.model_path, backend_, w_, moe_placement_, &hybrid_cfg, *hybrid, &err, expert_backend_)) { @@ -2059,6 +2431,7 @@ void DeepSeek4Backend::print_ready_banner() const { cfg_.paged_attention ? (int)paged_cache_.plan.max_ctx : cache_.max_ctx, w_.n_expert_used, w_.n_expert); + if (image_capable_) std::printf("[deepseek4] image input ready mmproj=%s\n", cfg_.mmproj_path.c_str()); std::fflush(stdout); } @@ -2100,6 +2473,7 @@ bool DeepSeek4Backend::park(ParkTarget target) { } moe_placement_ = {}; moe_decode_placement_ = {}; + vision_.reset(); free_deepseek4_weights(w_); parked_ = true; if (spec_drafter_) { @@ -2119,6 +2493,7 @@ bool DeepSeek4Backend::unpark(ParkTarget target) { if (want_target_model && parked_) { if (!load_model()) { std::fprintf(stderr, "[deepseek4] unpark: failed to restore target model\n"); + vision_.reset(); free_deepseek4_weights(w_); stream_engine_.destroy(); moe_hybrid_.reset(); @@ -2137,6 +2512,7 @@ bool DeepSeek4Backend::unpark(ParkTarget target) { "[deepseek4] unpark: failed to recreate KV cache (ctx=%d)\n", max_ctx); free_deepseek4_cache(cache_); + vision_.reset(); free_deepseek4_weights(w_); stream_engine_.destroy(); moe_hybrid_.reset(); @@ -2152,6 +2528,7 @@ bool DeepSeek4Backend::unpark(ParkTarget target) { if (env_flag_enabled("LUCE_DS4_MOE_TP") && !init_moe_tensor_parallel()) { free_deepseek4_cache(cache_); + vision_.reset(); free_deepseek4_weights(w_); expert_runtime_.reset(); stream_engine_.destroy(); @@ -2170,6 +2547,7 @@ bool DeepSeek4Backend::unpark(ParkTarget target) { std::fflush(stdout); } if (!validate_prefill_mode()) { + vision_.reset(); free_deepseek4_weights(w_); stream_engine_.destroy(); moe_hybrid_.reset(); @@ -2261,7 +2639,10 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, const DaemonIO & io, int kv_offset, int snap_slot, - int snap_pos) { + int snap_pos, + const DeepSeek4ImagePrompt * images) { + const bool capture_spec = !images && spec_enabled_ && spec_drafter_; + if (images) spec_feat_window_.clear(); const InferencePhase phase = deepseek4_roctx_prefill_phase( prefill_attention_mode_name(cfg_.prefill_mode)); const DeepSeek4RoctxPhaseScope roctx_phase(phase); @@ -2288,7 +2669,9 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, // Bound the layer-major graph to the topology validated by the prefill // kernels. Smaller tail chunks use the same scheduler or its reference // fallback. - const int layer_major_cap = DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS; + const int layer_major_cap = vision_ + ? std::min(1024, DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS) + : DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS; // Only sparse prefill has a qualified batched mixed-owner HC path. Dense // hybrid execution remains tokenwise; batching it would skip per-token HC // post-mixing and corrupt the hidden state. @@ -2321,15 +2704,35 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, base_chunk, chunk, kv_offset + n_total); } int pos = kv_offset; + const int image_capacity = std::min(1024, + deepseek4_hybrid_prefill_chunk_tokens(layer_major_cap, kv_offset + n_total)); const bool save_snapshot = - snap_slot >= 0 && snap_slot < PREFIX_SLOTS && + !images && snap_slot >= 0 && snap_slot < PREFIX_SLOTS && snap_pos > kv_offset && snap_pos <= kv_offset + n_total; + if (images) { + if (kv_offset != 0 || !images->matches(tokens)) return -1; + for (int offset = 0; offset < n_total;) { + const int proposed = deepseek4_hybrid_prefill_step_tokens(chunk, offset, n_total - offset); + const int count = vision::atomic_image_chunk(images->spans(), uint64_t(offset), + proposed, uint64_t(n_total - offset), image_capacity); + bool has_images = false; + std::string error; + if (!count || !deepseek4_validate_image_batch(w_, cache_, moe_hybrid_.get(), + tokens.data() + offset, count, offset, images->spans(), has_images, error)) { + std::fprintf(stderr, "[deepseek4] image chunk admission failed: %s\n", + error.empty() ? "complete image exceeds configured chunk capacity" : error.c_str()); + return -1; + } + offset += count; + } + } // New sequence: clear the cache buffer so compressor state double-buffers // and compressed-KV rows start from zeros, exactly like a fresh server. // Without this, the first flush windows of a request pool over the // previous request's leftover state rows and outputs from the 2nd/3rd // request on can drift by a token or two. if (kv_offset == 0) { + cache_has_images_ = images != nullptr; reset_deepseek4_cache(cache_); } last_logits_.clear(); @@ -2338,7 +2741,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, int spec_snap_from = n_total; int spec_snap_to = 0; int spec_old_rows_for_final = 0; - if (spec_enabled_ && spec_drafter_) { + if (capture_spec) { const int feat_row = spec_drafter_->n_target_layers * w_.n_embd; const int snap_tokens = save_snapshot ? snap_pos - kv_offset : n_total; spec_final_from = std::max(0, n_total - w_.n_swa); @@ -2399,7 +2802,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, // there and a full 256-expert duplicate stack makes that tail far more // expensive than slightly rebalancing the preceding chunk. const int tail_tokens = n_total - (i + n_tok); - if (moe_hybrid_ && spec_enabled_ && spec_drafter_ && + if (moe_hybrid_ && capture_spec && tail_tokens > 0 && tail_tokens < 512 && n_tok >= 1024 - tail_tokens) { n_tok -= 512 - tail_tokens; @@ -2410,7 +2813,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, snap_pos > pos && snap_pos < pos + n_tok) { n_tok = snap_pos - pos; } - if (spec_enabled_ && spec_drafter_) { + if (capture_spec) { const bool batch_final_capture = supports_batched_spec_feature_capture( w_.moe_hybrid, cache_.prefill_mode, n_tok); @@ -2420,6 +2823,12 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, spec_snap_from, spec_snap_to); } + if (images) { + n_tok = vision::atomic_image_chunk(images->spans(), uint64_t(pos), n_tok, + uint64_t(n_total - i), image_capacity); + if (!n_tok) return -1; + } + // Bulk prompt graphs and the final DSpark feature-capture graph have // different HC/owner arena shapes. Once all earlier chunks are // complete, retire their reusable prefill arenas before entering the @@ -2427,7 +2836,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, // than 300 MiB and needlessly makes the highest-throughput expert // placement fail only on the last chunk. const bool entering_final_capture_band = - spec_enabled_ && spec_drafter_ && i > 0 && + capture_spec && i > 0 && i + n_tok > spec_final_from; if (entering_final_capture_band && !capture_band_scratch_released) { @@ -2441,7 +2850,10 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, // Embed tokens std::vector embed(w_.n_embd * n_tok); const auto embed_t0 = Clock::now(); - w_.embedder.embed(tokens.data() + i, n_tok, embed.data()); + const bool embedded = images + ? images->embed_chunk(w_.embedder, size_t(i), n_tok, embed.data()) + : w_.embedder.embed(tokens.data() + i, n_tok, embed.data()); + if (!embedded) return -1; DeepSeek4StepTelemetry step_tel; if (timing) step_tel.embed_us = elapsed_us(embed_t0, Clock::now()); @@ -2461,7 +2873,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, const bool capture_snapshot = !snapshot_saved && i < spec_snap_to && i + n_tok > spec_snap_from; - if (spec_enabled_ && spec_drafter_ && + if (capture_spec && (capture_final || capture_snapshot)) { spec_hooks.capture_layer_ids = &spec_drafter_->capture_layer_ids; spec_hooks.capture_out = &spec_cap; @@ -2499,7 +2911,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, /*allow_decode_graph_reuse=*/true, hp, moe_hybrid_.get(), expert_runtime_.compute ? &expert_runtime_ : nullptr, - routing_stats_.get()); + routing_stats_.get(), images ? images->spans() : vision::ImageSpanView{}); } else if (moe_hybrid_) { ok = deepseek4_step(backend_, cfg_.device.gpu, w_, cache_, embed.data(), n_tok, pos, logits, moe_hybrid_.get(), tokens.data() + i, @@ -2516,7 +2928,10 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, need_logits ? &logits : nullptr, tokens.data() + i, timing ? &step_tel : nullptr, - cfg_.prefill_mode != PrefillAttentionMode::Sparse, hp); + cfg_.prefill_mode != PrefillAttentionMode::Sparse, hp, + /*moe_hybrid=*/nullptr, /*expert_runtime=*/nullptr, + /*routing_stats=*/nullptr, + images ? images->spans() : vision::ImageSpanView{}); } if (ok && hp && !spec_cap.empty()) { const int feat_row = spec_drafter_->n_target_layers * w_.n_embd; @@ -2552,7 +2967,7 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, std::fprintf(stderr, "[deepseek4] failed to save snapshot slot=%d pos=%d\n", snap_slot, snap_pos); - } else if (spec_enabled_ && spec_drafter_) { + } else if (capture_spec) { // Discard checkpoint-only rows once their snapshot is saved. // Retain just the already-captured prefix of the final SWA // window, so distant checkpoints do not bridge a huge gap in @@ -2566,10 +2981,24 @@ int DeepSeek4Backend::do_prefill(const std::vector & tokens, (size_t) final_new_rows); } } + // Completed chunks retain model/KV/HC state in backend buffers, not + // in the CUDA operator pool. Growing context-dependent temporaries + // can strand smaller legacy-pool blocks across successive chunks. + // Retire captured executables/memos and only returned pool blocks at + // this synchronized boundary; leave reusable gallocr arenas intact. + if (vision_ && bound_hybrid_scratch && n_tok >= 512 && + kv_offset + n_total > 4096 && + moe_hybrid_->cold_backend && + moe_hybrid_->cold_backend != backend_) { + ggml_backend_synchronize(backend_); + ggml_backend_synchronize(moe_hybrid_->cold_backend); + ggml_backend_cuda_trim_pool(backend_); + ggml_backend_cuda_trim_pool(moe_hybrid_->cold_backend); + } } keep_spec_feature_tail(spec_feat_window_, (size_t) std::max(0, w_.n_swa)); - if (timing && spec_enabled_ && spec_drafter_ && + if (timing && capture_spec && !spec_feat_window_.empty()) { size_t nonfinite = 0; double square_sum = 0.0; @@ -2768,13 +3197,38 @@ GenerateResult DeepSeek4Backend::generate_from_state( return result; } + const auto * images = dynamic_cast(req.images.get()); + if (req.images) { + if (!images || images->owner_ != this || !images->matches(req.prompt) || + kv_offset != 0 || req.snap_slot >= 0 || req.snap_pos >= 0 || + req.prompt.size() + uint64_t(std::max(0, req.n_gen)) > uint64_t(cache_.max_ctx) || + // Two-GPU serving needs its in-process second owner; one GPU has neither. + (moe_hybrid_ && !expert_backend_) || expert_runtime_.compute) { + result.fail(GenerateErrorCode::PrefillFailed, "image request binding, context, or execution mode is invalid"); + return result; + } + std::string error; + if (!materialize_images(*images, out_io, error)) { + if (out_io.is_cancelled()) { result.succeed(); return result; } + result.fail(GenerateErrorCode::PrefillFailed, error.empty() ? "image materialization failed" : error); + return result; + } + } else if (image_capable_ && + std::any_of(req.prompt.begin(), req.prompt.end(), [&](int32_t token) { + return token < 0 || token >= w_.n_vocab || + token == vision::ImageTokenizerContract{}.marker; + })) { + result.fail(GenerateErrorCode::PrefillFailed, "unbound image marker or invalid prompt token"); + return result; + } + // Prefill only the suffix that is not already represented by a restored // snapshot. An exact full-prompt hit can decode immediately from the // logits and speculative feature window saved with the cache state. int committed = kv_offset; if (kv_offset == 0) { committed = do_prefill(req.prompt, out_io, 0, - req.snap_slot, req.snap_pos); + req.snap_slot, req.snap_pos, images); } else if (kv_offset < (int) req.prompt.size()) { std::vector suffix(req.prompt.begin() + kv_offset, req.prompt.end()); @@ -2825,7 +3279,7 @@ GenerateResult DeepSeek4Backend::generate_from_state( } } if (spec_enabled_ && spec_drafter_ && req.n_gen > 0 && - !req.force_ar_decode && !budget_requires_ar && !sampling_requires_ar) { + !req.images && !req.force_ar_decode && !budget_requires_ar && !sampling_requires_ar) { if (last_logits_.empty()) { result.fail(GenerateErrorCode::DecodeFailed, "spec: no prefill logits"); return result; @@ -2910,6 +3364,7 @@ GenerateResult DeepSeek4Backend::generate_from_state( // ── Snapshots ─────────────────────────────────────────────────────────── bool DeepSeek4Backend::snapshot_save(int slot) { + if (cache_has_images_) return false; if (slot < 0 || slot >= PREFIX_SLOTS || !snap_backend_ || cache_.cur_pos <= 0 || last_logits_pos_ != cache_.cur_pos || w_.n_vocab <= 0 || @@ -3071,12 +3526,17 @@ bool DeepSeek4Backend::snapshot_restore(int slot) { last_logits_ = std::move(restored_logits); spec_feat_window_ = std::move(restored_features); last_logits_pos_ = cache_.cur_pos; + cache_has_images_ = false; return true; } GenerateResult DeepSeek4Backend::restore_and_generate_impl( int slot, const GenerateRequest & req, const DaemonIO & io) { GenerateResult result; + if (req.images) { + result.fail(GenerateErrorCode::PrefillFailed, "image requests cannot restore token-only snapshots"); + return result; + } if (!snapshot_used(slot)) { result.fail(GenerateErrorCode::InvalidSnapshotSlot); return result; @@ -3278,6 +3738,7 @@ void DeepSeek4Backend::shutdown() { routing_stats_out_path_.clear(); moe_placement_ = {}; moe_decode_placement_ = {}; + vision_.reset(); free_deepseek4_weights(w_); if (snap_backend_) { ggml_backend_free(snap_backend_); snap_backend_ = nullptr; } if (backend_) { ggml_backend_free(backend_); backend_ = nullptr; } diff --git a/server/src/deepseek4/deepseek4_backend.h b/server/src/deepseek4/deepseek4_backend.h index 2de4ef598..4e45ca153 100644 --- a/server/src/deepseek4/deepseek4_backend.h +++ b/server/src/deepseek4/deepseek4_backend.h @@ -15,6 +15,10 @@ #include "../common/moe_hybrid_stream.h" #include "deepseek4_internal.h" #include "deepseek4_dspark.h" +#include "deepseek4_vision.h" +#include "deepseek4_image_prompt.h" +#include "deepseek4_image_assembly.h" +#include "deepseek4_image_admission.h" #include "qwen3/qwen3_drafter.h" #include "deepseek4_seq_engine.h" @@ -28,6 +32,8 @@ namespace luce::common { +class DeepSeek4ImagePrompt; + // Bounds the sparse heterogeneous prefill arena once accumulated attention // context dominates its memory footprint. Decode batching is unaffected. int deepseek4_hybrid_prefill_chunk_tokens( @@ -66,6 +72,14 @@ class DeepSeek4Backend : public ModelBackend { // ModelBackend interface void print_ready_banner() const override; + bool supports_images() const override { return image_capable_ && vision_ != nullptr; } + std::string image_placeholder() const override { return vision::DS4V_IMAGE_PLACEHOLDER; } + bool prepare_images(std::vector & tokens, + std::vector images, + uint64_t context_capacity, + uint64_t output_reserve, + ImagePromptHandle & payload, + std::string & error) const override; bool park(ParkTarget target) override; bool unpark(ParkTarget target) override; @@ -114,6 +128,11 @@ class DeepSeek4Backend : public ModelBackend { DeepSeek4PagedCache paged_cache_; std::unique_ptr seq_engine_; bool parked_ = false; + bool image_capable_ = false; + bool cache_has_images_ = false; + std::unique_ptr vision_; + vision::ImageRequestGate image_request_gate_; + vision::ImageAdmissionReserves image_reserves_; // Sampler SamplerCfg sampler_; @@ -172,7 +191,12 @@ class DeepSeek4Backend : public ModelBackend { // Prefill prompt tokens in chunks, return absolute committed position. int do_prefill(const std::vector & tokens, const DaemonIO & io, - int kv_offset = 0, int snap_slot = -1, int snap_pos = -1); + int kv_offset = 0, int snap_slot = -1, int snap_pos = -1, + const DeepSeek4ImagePrompt * images = nullptr); + bool load_vision(); + bool init_single_gpu_vision(); + bool materialize_images(const DeepSeek4ImagePrompt & images, + const DaemonIO & io, std::string & error); // Generate after either a fresh prefill or a restored prefix. kv_offset is // the number of prompt tokens already represented by cache_ and the diff --git a/server/src/deepseek4/deepseek4_graph.cpp b/server/src/deepseek4/deepseek4_graph.cpp index 339d36089..cb91d7387 100644 --- a/server/src/deepseek4/deepseek4_graph.cpp +++ b/server/src/deepseek4/deepseek4_graph.cpp @@ -9,6 +9,9 @@ // 6. MoE FFN (hash routing + top-k + shared expert + clamped SwiGLU) #include "deepseek4_internal.h" +#include "deepseek4_norm.h" +#include "deepseek4_image_policy.h" +#include "deepseek4_vision.h" #include "common/blocking_row_pool.h" #include "deepseek4_hc_cuda.h" #include "deepseek4_roctx.h" @@ -346,7 +349,8 @@ static ggml_tensor * build_moe_ffn(ggml_context * ctx, const DeepSeek4Weights & w, const DeepSeek4Layer & L, int layer_idx, - int n_tokens); + int n_tokens, + ggml_tensor * selection_bias = nullptr); // Every cached per-layer decode/prefill graph below owns a StepGraph whose // metadata arena holds the ggml nodes the CUDA/HIP backend keys its captured @@ -707,8 +711,7 @@ static bool build_cached_decode_output_graph( static ggml_tensor * build_rms_norm(ggml_context * ctx, ggml_tensor * x, ggml_tensor * weight, float eps) { - ggml_tensor * normed = ggml_rms_norm(ctx, x, eps); - return ggml_mul(ctx, normed, weight); + return detail::build_rms_norm(ctx, x, weight, eps); } // ─── Helper: Clamped SwiGLU ───────────────────────────────────────────── @@ -2064,8 +2067,11 @@ static ggml_tensor * build_mla_output_projection( ctx, L.attn_output_a, group_dim, w.n_lora_o, w.n_out_group); ggml_tensor * attn_low = ds4_mul_mat_columns(ctx, out_a_3d, attn_out, projection_columns); + // The grouped source layout is read by MMQ's activation quantizer and by + // nothing else, so a projection stored unquantized (BF16 attention from a + // converter that leaves dense tensors alone) takes the plain path. const bool grouped_output_projection = - allow_grouped && n_tokens > 1 && + allow_grouped && n_tokens > 1 && ggml_is_quantized(L.attn_output_b->type) && !ds4_env_flag("LUCE_DS4_DISABLE_GROUPED_OUTPUT_PROJECTION"); if (grouped_output_projection) { return ggml_mul_mat_grouped_src(ctx, L.attn_output_b, attn_low); @@ -2144,7 +2150,8 @@ static ggml_tensor * build_mla_attention_lane_core( DeepSeek4AttentionImpl attention_impl = DeepSeek4AttentionImpl::Explicit, const DeepSeek4PreparedProjectedLane * prepared = nullptr, ggml_tensor ** out_attn_context = nullptr, - DeepSeek4SpecBoundaryCheckpointLayer * boundary_checkpoint = nullptr) { + DeepSeek4SpecBoundaryCheckpointLayer * boundary_checkpoint = nullptr, + vision::ImageSpanView image_spans = {}) { const int n_embd = w.n_embd; const int head_dim = w.head_dim; @@ -2209,7 +2216,7 @@ static ggml_tensor * build_mla_attention_lane_core( // steps on this path; LUCE_DS4_NO_CAUSAL_VERIFY=1 restores the legacy // (bidirectional) behavior for A/B comparison. const bool causal_batch = (n_tokens > 1) && !cached_inputs && f32_array_inputs && - !ds4_env_flag("LUCE_DS4_NO_CAUSAL_VERIFY"); + (image_spans.size || !ds4_env_flag("LUCE_DS4_NO_CAUSAL_VERIFY")); const bool layer_major_batch = causal_batch && attention_impl != DeepSeek4AttentionImpl::Explicit; ggml_tensor * old_rows_scratch = nullptr; @@ -2463,7 +2470,7 @@ static ggml_tensor * build_mla_attention_lane_core( constexpr int maskless_indexed_rows_cap = 512; // fattn.cu top-k scan width const bool maskless_sparse_prefill = attention_impl == DeepSeek4AttentionImpl::SparseFlash && - layer_major_batch && !gathered_history && + layer_major_batch && !gathered_history && !image_spans.size && indexer_topk && n_tokens > w.n_swa && indexer_topk->ne[0] <= maskless_indexed_rows_cap && n_prior_rows == std::min(kv_start, w.n_swa) && @@ -2660,11 +2667,12 @@ static ggml_tensor * build_mla_attention_lane_core( // [n_kv,n_query] F16; the explicit path broadcasts the same values over // heads in F32. ggml_tensor * score_mask = nullptr; + int raw_score_capacity = w.n_swa; // Ratio-4 sparse prefill already carries the authoritative compressed // row IDs. The CUDA/HIP kernel can derive the raw causal window and the // completed compressed-row frontier from kv_start and the query index. // Keep every other attention shape on the explicit mask contract. - const bool direct_indexer_topk = indexer_topk && + const bool direct_indexer_topk = indexer_topk && !image_spans.size && (maskless_sparse_prefill || ds4_env_flag("LUCE_DS4_DIRECT_INDEXER_TOPK")); // Layer-major, non-indexed layers can skip the quadratic causal mask: @@ -2701,7 +2709,7 @@ static ggml_tensor * build_mla_attention_lane_core( ds4_env_flag("GGML_CUDA_MLA_DENSE_HIGH_RATIO") && ds4_env_flag("GGML_CUDA_MLA_DENSE_WMMA"); const bool direct_contiguous_causal = layer_major_batch && - !gathered_history && !indexer_topk && n_tokens > w.n_swa && + !gathered_history && !indexer_topk && !image_spans.size && n_tokens > w.n_swa && (ratio == 0 || ratio > 1) && (analytic_causal_shmem <= analytic_causal_lds_limit || streaming_dense_high_ratio) && @@ -2728,15 +2736,29 @@ static ggml_tensor * build_mla_attention_lane_core( const int pos_i = kv_start + i; float * col = mvals.data() + (size_t) i * n_attn; const int min_pos = pos_i - w.n_swa + 1; + const auto * image = vision::image_block_at(image_spans, uint64_t(pos_i)); + const int64_t image_begin = image ? int64_t(image->visible_begin) : -1; + const int64_t image_end = image ? int64_t(image->visible_end) : -1; for (int r = 0; r < n_prior_rows; ++r) { const int prior_pos = kv_start - n_prior_rows + r; - if (prior_pos < min_pos) col[r] = -1e30f; + bool visible = prior_pos >= min_pos; + if (image_spans.size && !vision::raw_key_visible( + pos_i, prior_pos, w.n_swa, image_begin, image_end, visible)) return nullptr; + if (!visible) col[r] = -1e30f; } for (int t = 0; t < n_tokens; ++t) { const int current_pos = kv_start + t; - if (t > i || current_pos < min_pos) { - col[n_prior_rows + t] = -1e30f; + bool visible = t <= i && current_pos >= min_pos; + if (image_spans.size && !vision::raw_key_visible( + pos_i, current_pos, w.n_swa, image_begin, image_end, visible)) return nullptr; + if (!visible) col[n_prior_rows + t] = -1e30f; + } + if (image_spans.size) { + int first = n_raw, last = -1; + for (int r = 0; r < n_raw; ++r) { + if (col[r] == 0.0f) { first = std::min(first, r); last = r; } } + raw_score_capacity = std::max(raw_score_capacity, last - first + 1); } if (n_comp_attn > 0) { const int vis = gathered_history ? n_comp_attn @@ -2978,7 +3000,7 @@ static ggml_tensor * build_mla_attention_lane_core( // dense attention unchanged while allowing the D=512 value pass to // skip the two masked envelopes without guessing DS4 cache layout. ggml_flash_attn_ext_set_ds4_sparse( - context, n_raw, w.n_swa, + context, n_raw, raw_score_capacity, indexer_topk ? -(int) indexer_topk->ne[0] : attention_impl == DeepSeek4AttentionImpl::SparseFlash @@ -3137,7 +3159,8 @@ static ggml_tensor * build_mla_attention( std::vector & i64_array_inputs, std::vector * f32_array_inputs = nullptr, DeepSeek4AttentionImpl attention_impl = DeepSeek4AttentionImpl::Explicit, - DeepSeek4SpecBoundaryCheckpointLayer * boundary_checkpoint = nullptr) { + DeepSeek4SpecBoundaryCheckpointLayer * boundary_checkpoint = nullptr, + vision::ImageSpanView image_spans = {}) { const int ratio = w.compress_ratios[layer_idx]; DeepSeek4MlaLaneBindings lane = deepseek4_contiguous_lane_bindings( lc, ratio, kv_start + n_tokens - 1); @@ -3145,7 +3168,7 @@ static ggml_tensor * build_mla_attention( ctx, gf, cur, w, L, lane, layer_idx, kv_start, n_tokens, cached_inputs, i32_inputs, i32_array_inputs, i64_array_inputs, f32_array_inputs, attention_impl, /*prepared=*/nullptr, - /*out_attn_context=*/nullptr, boundary_checkpoint); + /*out_attn_context=*/nullptr, boundary_checkpoint, image_spans); } struct DeepSeek4CachedDecodeHcPreGraph { @@ -4005,12 +4028,16 @@ static bool eval_ds4_hybrid( return true; } +// `selection_bias`, when given, is a per-token [n_expert, n_tokens] input that +// replaces the layer's single selection bias. Image batches use it: image rows +// select with the image router bias, text rows keep their usual selection. static Ds4MoeRouting build_moe_routing( ggml_context * ctx, ggml_tensor * cur, const DeepSeek4Weights & w, const DeepSeek4Layer & L, - int n_tokens) { + int n_tokens, + ggml_tensor * selection_bias = nullptr) { Ds4MoeRouting out; auto track = [&](ggml_tensor * tensor) { if (tensor) out.nodes.push_back(tensor); @@ -4024,7 +4051,9 @@ static Ds4MoeRouting build_moe_routing( ggml_tensor * softplus = track(ggml_softplus(ctx, logits)); ggml_tensor * probs = track(ggml_sqrt(ctx, softplus)); ggml_tensor * selection = probs; - if (L.ffn_exp_probs_b) { + if (selection_bias) { + selection = track(ggml_add(ctx, selection, selection_bias)); + } else if (L.ffn_exp_probs_b) { selection = track(ggml_add(ctx, selection, L.ffn_exp_probs_b)); } @@ -4049,7 +4078,8 @@ static ggml_tensor * build_moe_ffn( const DeepSeek4Weights & w, const DeepSeek4Layer & L, int layer_idx, - int n_tokens) { + int n_tokens, + ggml_tensor * selection_bias) { const int n_embd = w.n_embd; int n_used = w.n_expert_used; @@ -4057,10 +4087,10 @@ static ggml_tensor * build_moe_ffn( ggml_tensor * shared_out = build_shared_ffn(ctx, cur, w, L); ggml_tensor * routed_out = nullptr; - if (layer_idx < w.n_hash_layer && L.ffn_gate_tid2eid) { + if (!selection_bias && layer_idx < w.n_hash_layer && L.ffn_gate_tid2eid) { routed_out = ggml_scale(ctx, cur, 0.0f); } else { - Ds4MoeRouting routing = build_moe_routing(ctx, cur, w, L, n_tokens); + Ds4MoeRouting routing = build_moe_routing(ctx, cur, w, L, n_tokens, selection_bias); n_used = (int) routing.selected->ne[0]; ggml_tensor * cur_3d = ggml_reshape_3d(ctx, cur, n_embd, 1, n_tokens); ggml_tensor * gate_e = ggml_mul_mat_id(ctx, L.ffn_gate_exps, cur_3d, routing.selected); @@ -6515,7 +6545,8 @@ static bool eval_ds4_layer_range_hybrid_ffn( MoeHybridRoutingStats * routing_stats, std::vector & out, DeepSeek4StepTelemetry * telemetry, - const MoeHybridDeviceOutputs * device_outputs = nullptr) { + const MoeHybridDeviceOutputs * device_outputs = nullptr, + int kv_start = 0, vision::ImageSpanView image_spans = {}) { const bool trace_prefill = ds4_env_flag("LUCE_DS4_PREFILL_TRACE"); if (trace_prefill) { std::fprintf(stderr, @@ -6684,6 +6715,13 @@ static bool eval_ds4_layer_range_hybrid_ffn( ggml_backend_tensor_get(L.ffn_exp_probs_b, bias.data(), 0, sizeof(float) * bias.size()); } + std::vector image_bias; + if (image_spans.size) { + if (!L.ffn_gate_bias_vl) return false; + image_bias.resize(size_t(w.n_expert)); + ggml_backend_tensor_get(L.ffn_gate_bias_vl, image_bias.data(), 0, + sizeof(float) * image_bias.size()); + } const auto route_select_t0 = Ds4TimingClock::now(); for (int t = 0; t < n_tokens; ++t) { @@ -6694,6 +6732,21 @@ static bool eval_ds4_layer_range_hybrid_ffn( float * token_weights = weights.data() + (size_t)t * (size_t)route_width; + if (vision::image_block_at(image_spans, uint64_t(kv_start + t))) { + vision::ImageExpertSelection selection; + std::string error; + if (!vision::select_image_experts(token_probs, image_bias.data(), + size_t(w.n_expert), size_t(route_width), selection, error, + w.expert_weight_scale)) { + std::fprintf(stderr, "[deepseek4] image routing failed: %s\n", error.c_str()); + return false; + } + std::copy_n(selection.indices.data(), route_width, token_ids_out); + std::copy_n(selection.weights.data(), route_width, token_weights); + observe_active_routing(routing_stats, layer, token_ids_out, token_weights, route_width); + continue; + } + if (hash_routed) { const int32_t tok = token_ids[t]; if (tok < 0 || tok >= w.n_vocab) return false; @@ -7071,12 +7124,16 @@ static int ds4_try_layer_major_prefill( std::vector * out_logits, const int32_t * token_ids, Ds4VerifyHooks * verify_hooks, - DeepSeek4StepTelemetry * telemetry) { + DeepSeek4StepTelemetry * telemetry, + vision::ImageSpanView image_spans = {}) { if (!backend || !embed || n_tokens <= 4 || n_tokens > DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS || kv_start < 0 || w.moe_hybrid) { return 0; } + // Image batches carry their own attention mask and per-token expert + // selection, so their graphs are built fresh and never cached. + const bool image_batch = image_spans.size != 0; if (cache.prefill_mode == PrefillAttentionMode::Exact) return 0; // Layer-major prefill returns only the final-position logits. DSpark's // per-layer feature capture is supported below, but verifier requests for @@ -7187,9 +7244,10 @@ static int ds4_try_layer_major_prefill( // growth costs several GiB without producing a cache hit. Keep the short // request win, then retire it before the first chunk beyond 32K. constexpr int layer_major_cache_context_limit = 32768; - const bool allow_graph_cache = + const bool cache_context_ok = token_ids && next_pos <= layer_major_cache_context_limit; - if (!allow_graph_cache) { + const bool allow_graph_cache = cache_context_ok && !image_batch; + if (!cache_context_ok) { for (auto & candidate : ds4_layer_major_graph_caches) { if (candidate.owner_ctx == w.ctx && candidate.backend == backend) { candidate.destroy(); @@ -7456,7 +7514,8 @@ static int ds4_try_layer_major_prefill( ggml_tensor * attn_out = build_mla_attention( ctx, gf, attn_normed, w, L, lc, il, kv_start, n_tokens, nullptr, i32_inputs, i32_array_inputs, i64_array_inputs, - &f32_array_inputs, attention_impl); + &f32_array_inputs, attention_impl, + /*boundary_checkpoint=*/nullptr, image_spans); if (!attn_out) { if (!cached_layer) ggml_free(ctx); return fail("attention graph build failed", il); @@ -7482,10 +7541,20 @@ static int ds4_try_layer_major_prefill( ggml_tensor * ffn_normed = build_rms_norm(ctx, ffn_in, L.ffn_norm, w.rms_eps); ggml_tensor * hash_ids = nullptr; + ggml_tensor * selection_bias = nullptr; ggml_tensor * ffn_out = nullptr; const bool hash_routed = il < w.n_hash_layer && L.ffn_gate_tid2eid && token_ids && hash_tables[(size_t) il].loaded; - if (hash_routed) { + if (image_batch) { + // One mechanism for every layer: top-k over probs + a per-token bias. + if (!L.ffn_gate_bias_vl) { + if (!cached_layer) ggml_free(ctx); + return fail("image batch without an image router bias", il); + } + selection_bias = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, w.n_expert, n_tokens); + ggml_set_input(selection_bias); + ffn_out = build_moe_ffn(ctx, ffn_normed, w, L, il, n_tokens, selection_bias); + } else if (hash_routed) { hash_ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, w.n_expert_used, n_tokens); ggml_set_input(hash_ids); @@ -7582,6 +7651,33 @@ static int ds4_try_layer_major_prefill( ggml_backend_tensor_set(hash_ids, hash_scratch.data(), 0, sizeof(int32_t) * hash_scratch.size()); } + if (selection_bias) { + // Image rows: the image router bias. Text rows: the layer's usual + // bias, or for a hash-routed layer a large bias on exactly the + // experts its table names, so top-k returns that set. + constexpr float HASH_PICK = 1.0e4f; + const size_t n_expert = (size_t) w.n_expert; + std::vector image_bias(n_expert), text_bias(n_expert, 0.0f); + ggml_backend_tensor_get(L.ffn_gate_bias_vl, image_bias.data(), 0, sizeof(float) * n_expert); + if (!hash_routed && L.ffn_exp_probs_b) { + ggml_backend_tensor_get(L.ffn_exp_probs_b, text_bias.data(), 0, sizeof(float) * n_expert); + } + std::vector bias(n_expert * (size_t) n_tokens); + for (int t = 0; t < n_tokens; ++t) { + float * row = bias.data() + (size_t) t * n_expert; + if (vision::image_block_at(image_spans, uint64_t(kv_start) + uint64_t(t))) { + std::copy(image_bias.begin(), image_bias.end(), row); + continue; + } + std::copy(text_bias.begin(), text_bias.end(), row); + if (hash_routed) { + const int32_t * picks = hash_tables[(size_t) il].ids.data() + + (size_t) token_ids[t] * (size_t) w.n_expert_used; + for (int k = 0; k < w.n_expert_used; ++k) row[picks[k]] = HASH_PICK; + } + } + ggml_backend_tensor_set(selection_bias, bias.data(), 0, sizeof(float) * bias.size()); + } if (telemetry) { telemetry->full_graph_build_us += ds4_elapsed_us( build_t0, Ds4TimingClock::now()); @@ -7734,6 +7830,68 @@ static bool initialize_layer_range_cache( runtime.owns_output = owns_output; return true; } +bool deepseek4_validate_image_batch( + const DeepSeek4Weights & w, const DeepSeek4Cache & cache, + const MoeHybridStorage * hybrid, const int32_t * tokens, + int count, int position, vision::ImageSpanView spans, + bool & has_images, std::string & error) { + has_images = false; + if (!spans.size) return true; + const auto fail = [&](const char * message) { error = message; return false; }; + if (count <= 0 || position < 0 || int64_t(position) + count > cache.max_ctx || + !vision::valid_image_spans(spans, uint64_t(std::max(0, cache.max_ctx)))) + return fail("invalid image batch bounds"); + const uint64_t end = uint64_t(position) + uint64_t(count); + for (size_t i = 0; i < spans.size; ++i) { + const auto & span = spans.data[i]; + if (span.block_begin >= end || span.block_end <= uint64_t(position)) continue; + if (span.block_begin < uint64_t(position) || span.block_end > end) + return fail("prefill batch would split an image block"); + has_images = true; + } + if (has_images && !tokens) return fail("image batch requires bound token IDs"); + if (tokens) { + for (int i = 0; i < count; ++i) { + const bool image_row = vision::image_block_at(spans, uint64_t(position) + uint64_t(i)); + const int32_t token = tokens[i]; + if (image_row ? (token < w.n_vocab || int64_t(token) >= int64_t(w.n_vocab) + 5) + : (token < 0 || token >= w.n_vocab)) + return fail("token IDs do not match image block positions"); + } + } + if (!has_images) return true; + // Images run through a batched sparse prefill: the single-GPU layer-major + // path, or the two-GPU path with both expert owners materialized on GPUs. + if (cache.prefill_mode != PrefillAttentionMode::Sparse || count <= 4 || + count > DS4_MAX_LAYER_MAJOR_PREFILL_TOKENS || + w.layers.size() != size_t(w.n_layer) || cache.layers.size() != size_t(w.n_layer) || + w.compress_ratios.size() != size_t(w.n_layer)) + return fail("image batch requires batched sparse prefill"); + if ((hybrid || w.moe_hybrid) && + (!hybrid || !w.moe_hybrid || !hybrid->materialized_cold_experts || + hybrid->cold_backend_kind != MoeHybridColdBackend::Gpu || !hybrid->cold_backend || + hybrid->layers.size() != size_t(w.n_layer))) + return fail("image batch requires both expert owners on GPUs"); + for (int il = 0; il < w.n_layer; ++il) { + const auto & layer = w.layers[size_t(il)]; + const auto & state = cache.layers[size_t(il)]; + const auto bias = layer.ffn_gate_bias_vl; + const int ratio = int(w.compress_ratios[size_t(il)]); + if (!bias || bias->type != GGML_TYPE_F32 || bias->ne[0] != w.n_expert || + ggml_nelements(bias) != w.n_expert || !state.raw_kv) + return fail("image decoder is missing a validated router bias or attention state"); + if (ratio && (!layer.attn_compressor_ape || !layer.attn_compressor_kv || + !layer.attn_compressor_gate || !layer.attn_compressor_norm || !state.comp_kv || + !state.attn_compressor.state_kv || !state.attn_compressor.state_score)) + return fail("image decoder has incomplete attention compressor state"); + if (ratio == 4 && (!layer.indexer_compressor_ape || !layer.indexer_compressor_kv || + !layer.indexer_compressor_gate || !layer.indexer_compressor_norm || + !state.index_comp_kv || !state.indexer_compressor.state_kv || + !state.indexer_compressor.state_score)) + return fail("image decoder has incomplete indexer compressor state"); + } + return true; +} struct Ds4PagedGatheredRuntime { DeepSeek4LayerRangeCache model; @@ -8137,9 +8295,23 @@ bool deepseek4_step_layer_range( Ds4VerifyHooks * verify_hooks, MoeHybridStorage * moe_hybrid, MoeExpertComputeRuntime * expert_runtime, - MoeHybridRoutingStats * routing_stats) { + MoeHybridRoutingStats * routing_stats, + vision::ImageSpanView image_spans) { const auto step_t0 = Ds4TimingClock::now(); + bool image_batch = false; + std::string image_error; + if (!deepseek4_validate_image_batch(w, cache, moe_hybrid, token_ids, + n_tokens, kv_start, image_spans, image_batch, image_error) || + (image_batch && (!embed || layer_begin != 0 || layer_end != w.n_layer || + verify_hooks || expert_runtime || + !vision::detail::hip_bias_workspace(backend) || + (moe_hybrid && moe_hybrid->cold_backend == backend)))) { + std::fprintf(stderr, "[deepseek4] image prefill rejected before evaluation: %s\n", + image_error.empty() ? "unsupported execution path" : image_error.c_str()); + return false; + } + if (!deepseek4_cuda_hc_set_device(device)) { std::fprintf(stderr, "[deepseek4] failed to select HC device %d for layer range [%d,%d)\n", @@ -8230,7 +8402,7 @@ bool deepseek4_step_layer_range( // to this forward call; decode graph replay is restored on every return. ScopedCudaGraphOverrides heterogeneous_prefill_eager_scope( heterogeneous_sparse_prefill && - ds4_env_flag("LUCE_DS4_HYBRID_PREFILL_EAGER")); + (image_batch || ds4_env_flag("LUCE_DS4_HYBRID_PREFILL_EAGER"))); // A dynamic batch may be supplied by callers other than the DSpark // verifier. Split it whenever it spans a learned-compressor boundary: @@ -8294,7 +8466,7 @@ bool deepseek4_step_layer_range( out_logits ? &chunk_out : nullptr, token_ids ? token_ids + off : nullptr, telemetry, allow_decode_graph_reuse, chunk_hooks_ptr, - moe_hybrid, expert_runtime, routing_stats)) { + moe_hybrid, expert_runtime, routing_stats, image_spans)) { return false; } hc_all.insert(hc_all.end(), chunk_hc.begin(), chunk_hc.end()); @@ -8439,6 +8611,16 @@ bool deepseek4_step_layer_range( moe_hybrid->prefill_cold_alloc = nullptr; } } + // Gallocr teardown does not return cached operator temporaries to + // the driver. Retire backend captures/memos and trim free pool blocks + // before allocating the new bulk-prefill scratch on either owner. + if (ds4_image_capable(w)) { + ggml_backend_cuda_trim_pool(backend); + if (moe_hybrid && moe_hybrid->cold_backend && + moe_hybrid->cold_backend != backend) { + ggml_backend_cuda_trim_pool(moe_hybrid->cold_backend); + } + } std::fprintf(stderr, "[deepseek4] released prior decode/tail arenas before " "new layer-major prefill\n"); @@ -8465,7 +8647,7 @@ bool deepseek4_step_layer_range( hc_layer_weights_range, hc_output_weights_range, hash_routing_tables_range, scratch.hash_expert_ids, embed, n_tokens, kv_start, out_logits, token_ids, verify_hooks, - telemetry); + telemetry, image_batch ? image_spans : vision::ImageSpanView{}); if (prc < 0) return false; if (prc > 0) { if (telemetry) { @@ -8475,6 +8657,11 @@ bool deepseek4_step_layer_range( return true; } } + // Only the two batched prefill paths know about image rows. + if (image_batch && !heterogeneous_sparse_prefill) { + std::fprintf(stderr, "[deepseek4] image prefill has no batched path for this configuration\n"); + return false; + } // The batched verifier graph is also the only whole-model graph that can // currently own tensors on both GPU backends. Reuse it for q=1 hybrid @@ -8978,7 +9165,10 @@ bool deepseek4_step_layer_range( i32_inputs, i32_array_inputs, i64_array_inputs, &f32_array_inputs, - attention_impl); + attention_impl, + /*boundary_checkpoint=*/nullptr, + image_batch ? image_spans : vision::ImageSpanView{}); + if (!attn_out) { ggml_free(ctx); return false; } ggml_set_output(attn_out); ggml_build_forward_expand(gf, attn_out); @@ -9460,7 +9650,8 @@ bool deepseek4_step_layer_range( token_ids, hash_routing_tables_range[(size_t)il], *moe_hybrid, expert_runtime, routing_stats, ffn_out_host, telemetry, - ffn_device_join ? &owner_outputs : nullptr)) { + ffn_device_join ? &owner_outputs : nullptr, + kv_start, image_batch ? image_spans : vision::ImageSpanView{})) { std::fprintf(stderr, "[deepseek4-moe-tp] layer-range FFN failed layer %d\n", il); @@ -9944,6 +10135,14 @@ void deepseek4_release_prefill_scratch( } } +void deepseek4_release_image_scratch(DeepSeek4Cache & c, + MoeHybridStorage * moe_hybrid) { + deepseek4_release_prefill_scratch(c, moe_hybrid); + delete c.layer_range_cache; + c.layer_range_cache = nullptr; + if (moe_hybrid) moe_hybrid->release_graph_caches(); +} + } // namespace luce::common // ══════════════════════════════════════════════════════════════════════ diff --git a/server/src/deepseek4/deepseek4_image_admission.cpp b/server/src/deepseek4/deepseek4_image_admission.cpp new file mode 100644 index 000000000..50afcfe9b --- /dev/null +++ b/server/src/deepseek4/deepseek4_image_admission.cpp @@ -0,0 +1,396 @@ +#include "deepseek4_image_admission.h" + +#include "deepseek4_internal.h" +#include "common/gpu_page_pool.h" +#include "common/moe_hybrid_placement.h" +#include "common/moe_hybrid_types.h" +#include "ggml-backend.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace luce::vision { +namespace { +constexpr uint64_t MAX = std::numeric_limits::max(); + +bool fail(std::string & error, const std::string & message) { + error = message; + return false; +} + +bool add(uint64_t & sum, uint64_t value) { + if (value > MAX - sum) return false; + sum += value; + return true; +} + +bool mul(uint64_t a, uint64_t b, uint64_t & result) { + if (a && b > MAX / a) return false; + result = a * b; + return true; +} + +bool aligned(uint64_t value, uint64_t alignment, uint64_t & result) { + if (!alignment) return false; + result = value; + return add(result, (alignment - value % alignment) % alignment); +} + +bool enabled(const char * name) { + const char * raw = std::getenv(name); + return raw && *raw && std::strcmp(raw, "0") != 0; +} + +bool gpu_device(ggml_backend_t backend) { + if (!backend || !ggml_backend_get_device(backend)) return false; + const auto type = ggml_backend_dev_type(ggml_backend_get_device(backend)); + return type == GGML_BACKEND_DEVICE_TYPE_GPU || type == GGML_BACKEND_DEVICE_TYPE_IGPU; +} + +bool selected_allocation(const ggml_tensor & source, uint64_t count, + ggml_backend_buffer_type_t buft, + uint64_t & payload, uint64_t & allocation, + std::string & error) { + payload = allocation = 0; + if (!count) return true; + if (!mul(uint64_t(source.nb[2]), count, payload) || payload > SIZE_MAX) { + return fail(error, "selected expert tensor payload overflow"); + } + // Storage recreates each owner tensor contiguously with only ne[2] changed. + // The metadata copy avoids allocating even a temporary GGML context. + ggml_tensor selected = source; + selected.ne[2] = int64_t(count); + selected.ne[3] = 1; + selected.nb[3] = size_t(payload); + selected.data = nullptr; + selected.buffer = nullptr; + selected.view_src = nullptr; + selected.view_offs = 0; + const uint64_t requested = ggml_backend_buft_get_alloc_size(buft, &selected); + if (requested < payload || + !aligned(requested, ggml_backend_buft_get_alignment(buft), allocation) || + allocation > ggml_backend_buft_get_max_size(buft)) { + return fail(error, "selected expert tensor allocation size is invalid or exceeds backend limit"); + } + return true; +} + +bool owner_activation_estimate(const common::MoeHybridConfig & config, int tokens, + uint64_t alignment, uint64_t & bytes, std::string & error) { + if (tokens <= 0 || tokens > 1024 || config.n_embd <= 0 || + config.n_ff_exp <= 0 || config.n_ff_shexp < 0 || config.n_expert_used <= 0) { + return fail(error, "image owner estimate requires positive dimensions and chunk capacity at most 1024"); + } + const uint64_t d = uint64_t(config.n_embd); + const uint64_t f = uint64_t(config.n_ff_exp); + const uint64_t shared_f = uint64_t(config.n_ff_shexp); + uint64_t pairs = 0, routed_width = 0, shared_width = 0, part = 0, elements = 0; + // Sum materialized tensors without lifetime reuse in the expert-major graph: + // packed input/output, matmul/scaled output, route gather (5*D per route); + // gate/up, optional scales/contiguous copies/clamps and GLU (<=9*F). + // Input/shared branch/reduction contributes <=4*D+9*shared_F per token. + // This intentionally leaves additional arena/pool/cached-graph headroom to + // the named reservation rather than inventing a backend-wide upper bound. + if (!mul(uint64_t(tokens), uint64_t(config.n_expert_used), pairs) || + !mul(5, d, routed_width) || !mul(9, f, part) || !add(routed_width, part) || + !mul(4, d, shared_width) || !mul(9, shared_f, part) || !add(shared_width, part) || + !mul(routed_width, pairs, elements) || !mul(shared_width, uint64_t(tokens), part) || + !add(elements, part) || !mul(elements, sizeof(float), bytes) || + !mul(12, pairs, part) || !add(bytes, part) || + !mul(16384, alignment, part) || !add(bytes, part)) { + return fail(error, "owner activation estimate overflow"); + } + return true; +} + +bool host_available(uint64_t & bytes, std::string & error) { +#if defined(__linux__) + std::ifstream input("/proc/meminfo"); + if (!input) return fail(error, "cannot read host MemAvailable"); + std::string line; + bool found = false; + while (std::getline(input, line)) { + if (line.compare(0, 13, "MemAvailable:") != 0) continue; + if (found) return fail(error, "duplicate host MemAvailable field"); + std::istringstream fields(line.substr(13)); + uint64_t kb = 0; + std::string unit, extra; + if (!(fields >> kb >> unit) || unit != "kB" || (fields >> extra) || + !mul(kb, 1024, bytes)) return fail(error, "invalid host MemAvailable"); + found = true; + } + if (!found) return fail(error, "host MemAvailable is missing"); + // MemAvailable leaves out pages the GPU driver holds for reuse. + if (!add(bytes, common::reclaimable_gpu_page_pool_bytes())) return fail(error, "host availability overflow"); + return true; +#else + (void) bytes; + return fail(error, "host admission currently requires Linux MemAvailable"); +#endif +} + +bool device_free(ggml_backend_t backend, ImageMemoryDomain domain, uint64_t & available, std::string & error) { + size_t free = 0, total = 0; + ggml_backend_dev_memory(ggml_backend_get_device(backend), &free, &total); + if (!total) return fail(error, "device memory query returned no capacity"); + // GGML's UMA query can legitimately report free RAM above dedicated total. + available = free; + // A host-shared device reports MemAvailable, which leaves out the pages the + // GPU driver holds for reuse. Dedicated VRAM has no such pool. + if (domain == ImageMemoryDomain::HostShared && + !add(available, common::reclaimable_gpu_page_pool_bytes())) return fail(error, "device availability overflow"); + return true; +} +} // namespace + +bool estimate_deepseek4_image_storage( + const common::DeepSeek4Weights & w, const common::MoeHybridPlacement & placement, + const common::MoeHybridConfig & config, ggml_backend_t primary, ggml_backend_t cold, + bool duplicate, ImageStorageEstimate & out, std::string & error) { + out = {}; + error.clear(); + if (!gpu_device(primary) || !gpu_device(cold) || + ggml_backend_get_device(primary) == ggml_backend_get_device(cold)) { + return fail(error, "image storage admission requires two distinct GPU owners"); + } + if (config.cold_expert_backend != common::MoeHybridColdBackend::Gpu || + !config.materialize_hot_experts || !config.materialize_cold_experts || + w.n_expert <= 0 || w.n_layer <= 0 || size_t(w.n_layer) != w.layers.size() || + config.n_layer != w.n_layer || config.n_expert != w.n_expert || + config.n_embd != w.n_embd || config.n_ff_exp != w.n_ff_exp || + config.n_expert_used != w.n_expert_used || !placement.matches(config) || + !placement.valid(&error)) { + return fail(error, "image storage admission requires valid materialized DS4 placement"); + } + if (duplicate != enabled("LUCE_MOE_DUPLICATE_HOT_ON_COLD")) { + return fail(error, "image admission duplicate mode differs from storage environment"); + } + const auto hot_buft = ggml_backend_get_default_buffer_type(primary); + const auto cold_buft = ggml_backend_get_default_buffer_type(cold); + if (!hot_buft || !cold_buft) return fail(error, "owner buffer type is missing"); + for (int layer_index = 0; layer_index < w.n_layer; ++layer_index) { + const auto & layer = w.layers[size_t(layer_index)]; + const uint64_t hot_count = uint64_t(placement.hot_counts[size_t(layer_index)]); + const uint64_t cold_count = duplicate ? uint64_t(w.n_expert) : uint64_t(w.n_expert) - hot_count; + if (enabled("LUCE_DS4_DECODE_ALL_COLD") && cold_count != uint64_t(w.n_expert)) { + return fail(error, "decode-all-cold requires a full physical cold stack before allocation"); + } + if ((layer.ffn_gate_shexp && layer.ffn_gate_shexp->ne[1] > config.n_ff_shexp) || + (layer.ffn_up_shexp && layer.ffn_up_shexp->ne[1] > config.n_ff_shexp)) { + return fail(error, "shared expert metadata exceeds the configured activation width"); + } + uint64_t hot_layer = 0, cold_layer = 0; + uint64_t hot_buffer = 0, cold_buffer = 0; + const auto count_buffer = [&](uint64_t allocation, uint64_t maximum, + uint64_t & current, uint64_t & count) { + if (current && allocation > maximum - current) { + if (!add(count, 1)) return false; + current = 0; + } + return add(current, allocation); + }; + const std::array surfaces{ + layer.ffn_gate_exps, layer.ffn_up_exps, layer.ffn_down_exps}; + for (const auto * tensor : surfaces) { + if (!tensor) continue; + if (tensor->ne[0] <= 0 || tensor->ne[1] <= 0 || tensor->ne[2] != w.n_expert || + tensor->ne[3] != 1 || tensor->nb[2] == 0 || !ggml_is_contiguous(tensor)) { + return fail(error, "expert metadata must be positive contiguous [in,out,E,1]"); + } + uint64_t full_payload = 0; + if (!mul(uint64_t(tensor->nb[2]), uint64_t(w.n_expert), full_payload) || + full_payload > SIZE_MAX || full_payload != ggml_nbytes(tensor)) { + return fail(error, "expert metadata payload overflow or stride mismatch"); + } + uint64_t hot_payload = 0, hot_alloc = 0, cold_payload = 0, cold_alloc = 0; + if (!selected_allocation(*tensor, hot_count, hot_buft, hot_payload, hot_alloc, error) || + !selected_allocation(*tensor, cold_count, cold_buft, cold_payload, cold_alloc, error)) return false; + if (!add(out.hot_payload_bytes, hot_payload) || !add(out.cold_payload_bytes, cold_payload) || + !add(hot_layer, hot_alloc) || !add(cold_layer, cold_alloc)) { + return fail(error, "expert owner storage sum overflow"); + } + if (!count_buffer(hot_alloc, ggml_backend_buft_get_max_size(hot_buft), + hot_buffer, out.hot_buffer_count) || + !count_buffer(cold_alloc, ggml_backend_buft_get_max_size(cold_buft), + cold_buffer, out.cold_buffer_count)) { + return fail(error, "owner split-buffer accounting overflow"); + } + out.largest_copy_bytes = std::max({out.largest_copy_bytes, hot_payload, cold_payload}); + uint64_t per_expert_table = 0; + if (tensor->type == GGML_TYPE_Q3_1_ROCMFP3_MIX) per_expert_table = 33; + if (tensor->type == GGML_TYPE_Q2_1_ROCMFP2_MIX) per_expert_table = 17; + if (per_expert_table) { + uint64_t hot_table = 0, cold_table = 0, host_table = 0; + if (!mul(per_expert_table, hot_count, hot_table) || + !mul(per_expert_table, cold_count, cold_table) || + !add(out.hot_mix_table_bytes, hot_table) || !add(out.cold_mix_table_bytes, cold_table) || + !mul(per_expert_table + (per_expert_table == 33 ? 1 : 0), + uint64_t(w.n_expert), host_table) || + !add(host_table, std::max(hot_table, cold_table)) || + !add(host_table, uint64_t(w.n_expert))) { + return fail(error, "MIX table accounting overflow"); + } + // File entry books/modes (+ P4 rotation bytes), compact selected + // books/modes, and conservatively one byte per seen expert. + out.host_mix_payload_peak_bytes = std::max(out.host_mix_payload_peak_bytes, host_table); + if (!add(out.mix_device_allocation_count, 2 * uint64_t(hot_count > 0) + + 2 * uint64_t(cold_count > 0))) { + return fail(error, "MIX allocation count overflow"); + } + } + } + if (!add(out.hot_allocation_bytes, hot_layer) || !add(out.cold_allocation_bytes, cold_layer) || + !add(out.hot_buffer_count, uint64_t(hot_buffer > 0)) || + !add(out.cold_buffer_count, uint64_t(cold_buffer > 0))) { + return fail(error, "owner allocation accounting overflow"); + } + } + if (!out.hot_payload_bytes && !out.cold_payload_bytes) return fail(error, "no expert storage metadata found"); + // The qualified Linux libstdc++ vector grows to at most old_size + max(old_size, + // requested_delta). Retained old storage during growth adds at most another M. + // Hot and cold staging vectors have disjoint scopes; layers are copied serially. +#if defined(__GLIBCXX__) + if (!mul(out.largest_copy_bytes, 3, out.host_copy_peak_bytes)) { + return fail(error, "host expert staging peak overflow"); + } +#else + return fail(error, "host expert staging growth bound requires the qualified libstdc++ runtime"); +#endif + return true; +} + +bool check_deepseek4_image_admission( + const common::DeepSeek4Weights & w, const common::MoeHybridPlacement & placement, + const common::MoeHybridConfig & config, ggml_backend_t primary, ggml_backend_t cold, + const ImageAdmissionReserves & reserves, ImageAdmissionReport & out, std::string & error) { + out = {}; + if (!estimate_deepseek4_image_storage(w, placement, config, primary, cold, + reserves.duplicate_hot_on_cold, out.storage, error)) return false; + out.storage_estimated = true; + out.cold_runtime_reservation_bytes = reserves.cold_runtime_reservation_bytes; + if (!owner_activation_estimate(config, reserves.max_chunk_tokens, + ggml_backend_buft_get_alignment(ggml_backend_get_default_buffer_type(cold)), + out.cold_activation_estimate_bytes, error)) return false; + if (reserves.primary_domain == ImageMemoryDomain::Unknown || + reserves.cold_domain == ImageMemoryDomain::Unknown) { + return fail(error, "actual owner host-memory sharing must be classified before admission"); + } + if (!device_free(primary, reserves.primary_domain, out.primary_free_bytes, error) || + !device_free(cold, reserves.cold_domain, out.cold_free_bytes, error) || + !host_available(out.host_available_bytes, error)) return false; + const ImageStorageEstimate storage = out.storage; + const uint64_t activation_bytes = out.cold_activation_estimate_bytes; + const ImageMemorySnapshot snapshot{out.primary_free_bytes, out.cold_free_bytes, out.host_available_bytes}; + return assess_deepseek4_image_admission(storage, activation_bytes, reserves, snapshot, out, error); +} + +bool check_deepseek4_image_host_preparation(uint64_t required_bytes, std::string & error) { + error.clear(); + if (!required_bytes) return fail(error, "image preparation requires a nonzero host reservation"); + uint64_t available = 0; + if (!host_available(available, error)) return false; + if (required_bytes > available) { + return fail(error, "insufficient host memory for image preparation: required=" + + std::to_string(required_bytes) + " available=" + std::to_string(available)); + } + return true; +} + +bool check_deepseek4_image_single_gpu_admission( + ggml_backend_t gpu, ImageMemoryDomain domain, uint64_t required_bytes, + uint64_t & free_bytes, std::string & error) { + error.clear(); + free_bytes = 0; + if (!gpu_device(gpu)) return fail(error, "image admission requires a GPU owner"); + if (domain == ImageMemoryDomain::Unknown) return fail(error, "the owner's memory domain must be classified"); + if (!device_free(gpu, domain, free_bytes, error)) return false; + if (required_bytes > free_bytes) return fail(error, "insufficient GPU headroom for image scratch"); + return true; +} + +bool check_deepseek4_image_runtime_admission( + const common::MoeHybridConfig & config, ggml_backend_t primary, ggml_backend_t cold, + const ImageAdmissionReserves & reserves, ImageAdmissionReport & out, std::string & error) { + out = {}; + error.clear(); + if (!gpu_device(primary) || !gpu_device(cold) || + ggml_backend_get_device(primary) == ggml_backend_get_device(cold)) { + return fail(error, "image runtime admission requires two distinct GPU owners"); + } + uint64_t activation_bytes = 0; + const auto cold_buft = ggml_backend_get_default_buffer_type(cold); + if (!cold_buft || !owner_activation_estimate(config, reserves.max_chunk_tokens, + ggml_backend_buft_get_alignment(cold_buft), activation_bytes, error)) return false; + ImageMemorySnapshot snapshot; + if (!device_free(primary, reserves.primary_domain, snapshot.primary_free_bytes, error) || + !device_free(cold, reserves.cold_domain, snapshot.cold_free_bytes, error) || + !host_available(snapshot.host_available_bytes, error)) return false; + ImageAdmissionReserves runtime_reserves = reserves; + runtime_reserves.host_loader_overhead_bytes = 0; + return assess_deepseek4_image_admission({}, activation_bytes, runtime_reserves, snapshot, out, error); +} + +bool assess_deepseek4_image_admission( + const ImageStorageEstimate & storage, uint64_t cold_activation_estimate_bytes, + const ImageAdmissionReserves & reserves, const ImageMemorySnapshot & snapshot, + ImageAdmissionReport & out, std::string & error) { + const ImageStorageEstimate storage_copy = storage; + out = {}; + error.clear(); + out.storage = storage_copy; + out.storage_estimated = true; + out.cold_activation_estimate_bytes = cold_activation_estimate_bytes; + out.cold_runtime_reservation_bytes = reserves.cold_runtime_reservation_bytes; + out.primary_free_bytes = snapshot.primary_free_bytes; + out.cold_free_bytes = snapshot.cold_free_bytes; + out.host_available_bytes = snapshot.host_available_bytes; + const auto known_domain = [](ImageMemoryDomain domain) { + return domain == ImageMemoryDomain::Dedicated || domain == ImageMemoryDomain::HostShared; + }; + if (!known_domain(reserves.primary_domain) || !known_domain(reserves.cold_domain)) { + return fail(error, "actual owner host-memory sharing must be classified before admission"); + } + out.primary_required_bytes = out.storage.hot_allocation_bytes; + out.cold_required_bytes = out.storage.cold_allocation_bytes; + if (!add(out.primary_required_bytes, out.storage.hot_mix_table_bytes) || + !add(out.primary_required_bytes, reserves.primary_future_bytes) || + !add(out.cold_required_bytes, out.storage.cold_mix_table_bytes) || + !add(out.cold_required_bytes, reserves.cold_future_bytes) || + !add(out.cold_required_bytes, reserves.cold_runtime_reservation_bytes)) { + return fail(error, "device admission charge overflow"); + } + uint64_t loading = out.storage.host_copy_peak_bytes; + if (!add(loading, out.storage.host_mix_payload_peak_bytes) || + !add(loading, reserves.host_loader_overhead_bytes)) { + return fail(error, "host model-loading charge overflow"); + } + // Loading and serving are separate phases. Conservatively keep runtime + // host buffers alongside whichever phase has the larger temporary peak. + out.host_required_bytes = std::max(loading, reserves.host_request_bytes); + if (!add(out.host_required_bytes, reserves.host_runtime_bytes) || + (reserves.primary_domain == ImageMemoryDomain::HostShared && + !add(out.host_required_bytes, out.primary_required_bytes)) || + (reserves.cold_domain == ImageMemoryDomain::HostShared && + !add(out.host_required_bytes, out.cold_required_bytes))) { + return fail(error, "combined host/UMA admission charge overflow"); + } + out.known_charges_fit = out.primary_required_bytes <= out.primary_free_bytes && + out.cold_required_bytes <= out.cold_free_bytes && out.host_required_bytes <= out.host_available_bytes; + if (!out.known_charges_fit) return fail(error, "insufficient primary, cold-owner, or combined host/UMA headroom"); + if (reserves.cold_runtime_reservation_bytes < out.cold_activation_estimate_bytes) { + return fail(error, "cold-owner runtime reservation is below the metadata-derived activation estimate"); + } + const bool loading_storage = storage_copy.hot_allocation_bytes || storage_copy.cold_allocation_bytes || + storage_copy.hot_mix_table_bytes || storage_copy.cold_mix_table_bytes || + storage_copy.host_copy_peak_bytes || storage_copy.host_mix_payload_peak_bytes; + if (!reserves.host_request_bytes || (loading_storage && !reserves.host_loader_overhead_bytes)) { + return fail(error, "request/decode and metadata/host-safety reservations are required"); + } + return true; +} +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_image_admission.h b/server/src/deepseek4/deepseek4_image_admission.h new file mode 100644 index 000000000..12378d718 --- /dev/null +++ b/server/src/deepseek4/deepseek4_image_admission.h @@ -0,0 +1,131 @@ +#pragma once + +#include +#include + +struct ggml_backend; +namespace luce::common { +struct DeepSeek4Weights; +struct MoeHybridPlacement; +struct MoeHybridConfig; +} + +namespace luce::vision { + +enum class ImageMemoryDomain { Unknown, Dedicated, HostShared }; + +struct ImageAdmissionReserves { + // GGML currently reports integrated HIP devices as GPU. The caller must + // classify the actual devices using HIP properties and unified-memory mode. + ImageMemoryDomain primary_domain = ImageMemoryDomain::Unknown; + ImageMemoryDomain cold_domain = ImageMemoryDomain::Unknown; + bool duplicate_hot_on_cold = false; + + // Only allocations not already reflected by the free-memory snapshot: + // KV, the remaining vision reservation, runtime arenas/pools, and existing + // allocator/safety margins. Do not charge preloaded core/projector twice. + uint64_t primary_future_bytes = 0; + uint64_t cold_future_bytes = 0; + // Named cold-runtime arena/pool reservation, additional to cold_future_bytes. + // This is a conservative headroom policy tested by the resource guard, not + // a claim that every backend temporary has a formal upper bound. + uint64_t cold_runtime_reservation_bytes = 0; + int max_chunk_tokens = 1024; + + // Full request/decode/preprocess host peak for the configured request limits + // and concurrency, plus any future host-only runtime/staging allocations. + uint64_t host_request_bytes = 0; + // GGUF metadata parser copies, embedded-table backing, allocator overhead, + // and the existing host safety reserve. Owner table payloads and expert + // copy staging are computed separately below. + uint64_t host_loader_overhead_bytes = 0; + uint64_t host_runtime_bytes = 0; +}; + +struct ImageStorageEstimate { + uint64_t hot_payload_bytes = 0; + uint64_t cold_payload_bytes = 0; + uint64_t hot_allocation_bytes = 0; + uint64_t cold_allocation_bytes = 0; + uint64_t hot_mix_table_bytes = 0; + uint64_t cold_mix_table_bytes = 0; + uint64_t largest_copy_bytes = 0; + uint64_t host_copy_peak_bytes = 0; + uint64_t host_mix_payload_peak_bytes = 0; + uint64_t hot_buffer_count = 0; + uint64_t cold_buffer_count = 0; + uint64_t mix_device_allocation_count = 0; +}; + +struct ImageAdmissionReport { + ImageStorageEstimate storage; + uint64_t primary_free_bytes = 0; + uint64_t cold_free_bytes = 0; + uint64_t host_available_bytes = 0; + uint64_t primary_required_bytes = 0; + uint64_t cold_required_bytes = 0; + uint64_t host_required_bytes = 0; + bool storage_estimated = false; + bool known_charges_fit = false; + uint64_t cold_activation_estimate_bytes = 0; + uint64_t cold_runtime_reservation_bytes = 0; +}; + +struct ImageMemorySnapshot { + uint64_t primary_free_bytes = 0; + uint64_t cold_free_bytes = 0; + uint64_t host_available_bytes = 0; +}; + +// Pure assessment shared by live admission and deterministic CPU tests. All +// figures describe allocations still to come relative to this one snapshot. +bool assess_deepseek4_image_admission( + const ImageStorageEstimate & storage, uint64_t cold_activation_estimate_bytes, + const ImageAdmissionReserves & reserves, const ImageMemorySnapshot & snapshot, + ImageAdmissionReport & result, std::string & error); + +// Reads metadata and backend allocation-size functions only; no device buffers, +// expert copies, graph execution, file mapping, or table registration occur. +// Supported scope matches DS4's materialized same-runtime two-GPU owner path +// with zero cache slots. Duplicate mode must match the storage environment. +bool estimate_deepseek4_image_storage( + const common::DeepSeek4Weights & weights, + const common::MoeHybridPlacement & placement, + const common::MoeHybridConfig & config, + ggml_backend * primary, ggml_backend * cold, + bool duplicate_hot_on_cold, ImageStorageEstimate & result, std::string & error); + +// Invoke after owner initialization and final placement, immediately before +// build_deepseek4_moe_hybrid_storage_from_file_with_mmap. Queries current device +// free memory and Linux MemAvailable, combining new UMA and host charges. +// Returns false on missing reservations or insufficient capacity. The report +// separates the metadata-derived activation estimate from reserved headroom. +// A successful snapshot is admission, not an allocation reservation or a proof +// against concurrent external allocations; guarded runtime verification remains +// necessary. Host cgroup/process limits must be included by caller policy. +bool check_deepseek4_image_admission( + const common::DeepSeek4Weights & weights, + const common::MoeHybridPlacement & placement, + const common::MoeHybridConfig & config, + ggml_backend * primary, ggml_backend * cold, + const ImageAdmissionReserves & reserves, + ImageAdmissionReport & result, std::string & error); + +// CPU-only predecode check for the client thread after taking the image lease. +// Does not touch backend state or query a GPU. +bool check_deepseek4_image_host_preparation(uint64_t required_bytes, std::string & error); + +// Recheck after synchronizing owners and releasing disposable graph caches. +// Existing experts/tables/core/KV/snapshots are represented only by live free +// memory. No model-loader or expert-copy charge is added a second time. +// One GPU holds the whole model: `required_bytes` of image scratch and future +// KV must fit in what that GPU has free right now. +bool check_deepseek4_image_single_gpu_admission( + ggml_backend * gpu, ImageMemoryDomain domain, uint64_t required_bytes, + uint64_t & free_bytes, std::string & error); + +bool check_deepseek4_image_runtime_admission( + const common::MoeHybridConfig & config, ggml_backend * primary, ggml_backend * cold, + const ImageAdmissionReserves & reserves, ImageAdmissionReport & result, std::string & error); + +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_image_assembly.cpp b/server/src/deepseek4/deepseek4_image_assembly.cpp new file mode 100644 index 000000000..7a25ad456 --- /dev/null +++ b/server/src/deepseek4/deepseek4_image_assembly.cpp @@ -0,0 +1,203 @@ +#include "deepseek4_image_assembly.h" +#include "deepseek4_image_spans.h" +#include +#include +#include +#include +#include + +namespace luce::vision { +namespace { +void require(bool valid, const char * message) { + if (!valid) throw std::runtime_error(message); +} +size_t elements(size_t rows, size_t dimension) { + require(dimension && rows <= std::numeric_limits::max() / dimension, + "invalid image matrix dimensions"); + return rows * dimension; +} +bool finite(const std::vector & values) { + return std::all_of(values.begin(), values.end(), [](float v) { return std::isfinite(v); }); +} +void layout_valid(const ImageLayout & layout) { + const auto & span = layout.span; + require(valid_image_spans({&span, 1}, span.block_end) && + span.block_end - span.block_begin == layout.types.size(), "invalid image layout span"); + size_t images = 0; + for (size_t row = 0; row < layout.types.size(); ++row) { + const uint64_t pos = span.block_begin + row; + const auto type = layout.types[row]; + if (pos == span.visible_begin) { + require(type == ImageTokenType::Start, "image start identity mismatch"); + } else if (pos + 1 == span.visible_end) { + require(type == ImageTokenType::End, "image end identity mismatch"); + } else if (pos < span.visible_begin || pos >= span.visible_end) { + require(type == ImageTokenType::Pad, "image outer padding identity mismatch"); + } else { + require(type == ImageTokenType::Pad || type == ImageTokenType::Newline || + type == ImageTokenType::Image, "invalid image token identity"); + } + images += type == ImageTokenType::Image; + } + require(images > 0 && images == layout.permutation.size(), "image permutation cardinality mismatch"); + std::vector seen(images, false); + for (int64_t index : layout.permutation) { + require(index >= 0 && uint64_t(index) < images && !seen[size_t(index)], + "image permutation must contain each raster row exactly once"); + seen[size_t(index)] = true; + } +} +void sentinels_valid(const ImageSentinels & s, size_t dimension) { + for (const auto * row : {&s.start, &s.pad, &s.newline, &s.end}) { + require(row->size() == dimension && dimension && finite(*row), "invalid image sentinel row"); + } +} +void cancelled(const ImageCancelled & callback) { + require(!callback || !callback(), "image materialization cancelled"); +} +} // namespace + +std::shared_ptr ImageRequestGate::try_acquire() const { + bool expected = false; + if (!active_->compare_exchange_strong(expected, true, std::memory_order_acq_rel)) return {}; + // shared_ptr invokes the deleter if control-block allocation throws too. + return std::shared_ptr(active_.get(), [active = active_](void *) { + active->store(false, std::memory_order_release); + }); +} + +bool assemble_image_rows(const ImageLayout & layout, const ImageRaster & raster, + const ImageSentinels & sentinels, size_t dimension, + std::vector & output, std::string & error) { + error.clear(); + try { + layout_valid(layout); + sentinels_valid(sentinels, dimension); + require(raster.rows == layout.permutation.size() && raster.columns == dimension && + raster.values.size() == elements(raster.rows, dimension) && finite(raster.values), + "invalid projected image raster"); + std::vector result(elements(layout.types.size(), dimension)); + size_t index = 0; + for (size_t row = 0; row < layout.types.size(); ++row) { + const float * source = nullptr; + switch (layout.types[row]) { + case ImageTokenType::Start: source = sentinels.start.data(); break; + case ImageTokenType::Pad: source = sentinels.pad.data(); break; + case ImageTokenType::Newline: source = sentinels.newline.data(); break; + case ImageTokenType::End: source = sentinels.end.data(); break; + case ImageTokenType::Image: + source = raster.values.data() + size_t(layout.permutation[index++]) * dimension; + break; + } + require(source != nullptr, "unknown image token type"); + std::copy_n(source, dimension, result.data() + row * dimension); + } + output.swap(result); + return true; + } catch (const std::exception & e) { error = e.what(); return false; } + catch (...) { error = "image assembly failed"; return false; } +} + +bool materialize_image_rows(const std::vector & images, + const ImageSentinels & sentinels, size_t dimension, + const ImageEncode & encode, const ImageCancelled & is_cancelled, + ImageRows & output, std::string & error) { + error.clear(); + try { + require(!images.empty() && images.size() <= 4 && bool(encode), "invalid image encode request"); + cancelled(is_cancelled); + sentinels_valid(sentinels, dimension); + uint64_t previous_end = 0; + for (const auto & image : images) { + layout_valid(image.layout); + require(image.input.plan.aligner_rows && image.input.plan.aligner_cols && + uint64_t(image.input.plan.aligner_rows) * image.input.plan.aligner_cols == + image.layout.permutation.size(), "prepared aligner shape does not match raster rows"); + require(image.layout.span.block_begin >= previous_end, "overlapping image blocks"); + previous_end = image.layout.span.block_end; + } + ImageRows result; + result.reserve(images.size()); + for (const auto & image : images) { + cancelled(is_cancelled); + ImageRaster raster; + if (!encode(image, raster, error)) { + if (error.empty()) error = "image encode failed"; + return false; + } + cancelled(is_cancelled); + std::vector rows; + if (!assemble_image_rows(image.layout, raster, sentinels, dimension, rows, error)) return false; + result.push_back(std::move(rows)); + } + cancelled(is_cancelled); + output.swap(result); + return true; + } catch (const std::exception & e) { error = e.what(); return false; } + catch (...) { error = "image encode callback failed"; return false; } +} + +bool embed_image_prompt_chunk(const PreparedImagePrompt & prompt, const ImageRows & rows, + int32_t vocabulary, size_t dimension, size_t position, + size_t count, const TextEmbed & embed, + std::vector & output, std::string & error) { + error.clear(); + try { + require(bool(prompt) && vocabulary > 0 && count && position <= prompt.tokens.size() && + count <= prompt.tokens.size() - position && rows.size() == prompt.images.size() && + prompt.images.size() <= 4, "invalid mixed embedding request"); + const size_t end = position + count; + uint64_t previous_end = 0; + for (size_t i = 0; i < prompt.images.size(); ++i) { + const auto & layout = prompt.images[i].layout; + const auto & span = layout.span; + layout_valid(layout); + require(span.block_begin >= previous_end && span.block_end <= prompt.tokens.size(), + "invalid mixed embedding spans"); + previous_end = span.block_end; + if (span.block_end <= position || span.block_begin >= end) continue; + require(span.block_begin >= position && span.block_end <= end, + "mixed embedding chunk splits an image block"); + require(rows[i].size() == elements(layout.types.size(), dimension) && finite(rows[i]), + "invalid materialized image matrix"); + for (size_t r = 0; r < layout.types.size(); ++r) { + const int64_t expected = int64_t(vocabulary) + int64_t(layout.types[r]); + require(int64_t(prompt.tokens[size_t(span.block_begin) + r]) == expected, + "external token does not match image layout"); + } + } + // Validate every ordinary ID before any callback can observe this chunk. + size_t image_index = 0; + for (size_t p = position; p < end; ++p) { + while (image_index < prompt.images.size() && prompt.images[image_index].layout.span.block_end <= p) + ++image_index; + if (image_index < prompt.images.size() && prompt.images[image_index].layout.span.block_begin <= p) continue; + require(prompt.tokens[p] >= 0 && prompt.tokens[p] < vocabulary, "unbound external token in text range"); + } + std::vector result(elements(count, dimension)); + size_t current = position; + image_index = 0; + while (current < end) { + while (image_index < prompt.images.size() && prompt.images[image_index].layout.span.block_end <= current) + ++image_index; + const size_t text_end = image_index < prompt.images.size() + ? std::min(end, size_t(prompt.images[image_index].layout.span.block_begin)) : end; + if (current < text_end) { + require(bool(embed) && embed(prompt.tokens.data() + current, text_end - current, + result.data() + (current - position) * dimension), "text embedding failed"); + current = text_end; + } + if (current == end) break; + const auto & block = prompt.images[image_index].layout.span; + std::copy(rows[image_index].begin(), rows[image_index].end(), + result.data() + (current - position) * dimension); + current = size_t(block.block_end); + } + require(finite(result), "nonfinite mixed embeddings"); + output.swap(result); + return true; + } catch (const std::exception & e) { error = e.what(); return false; } + catch (...) { error = "text embedding callback failed"; return false; } +} + +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_image_assembly.h b/server/src/deepseek4/deepseek4_image_assembly.h new file mode 100644 index 000000000..f343a6a41 --- /dev/null +++ b/server/src/deepseek4/deepseek4_image_assembly.h @@ -0,0 +1,50 @@ +#pragma once + +#include "deepseek4_image_prompt.h" +#include +#include +#include + +namespace luce::vision { + +// Bound decoded/prepared image memory to one outstanding request. The lease +// travels with the immutable payload and may be released by another thread. +class ImageRequestGate { +public: + std::shared_ptr try_acquire() const; +private: + std::shared_ptr> active_ = std::make_shared>(false); +}; + +struct ImageRaster { + size_t rows = 0, columns = 0; + std::vector values; +}; +struct ImageSentinels { + std::vector start, pad, newline, end; +}; +using ImageRows = std::vector>; +using ImageEncode = std::function; +using ImageCancelled = std::function; +using TextEmbed = std::function; + +// All outputs are replaced only on success. The permutation maps each Image +// token to one unique raster row; sentinel identities never use enum casts. +bool assemble_image_rows(const ImageLayout &, const ImageRaster &, const ImageSentinels &, + size_t dimension, std::vector & output, std::string & error); + +// Synchronous callbacks; cancellation is checked before/after every encode +// and before commit. Failure, cancellation, or callback exception leaves the +// caller's previous result intact. The caller owns GPU lifetime and scratch. +bool materialize_image_rows(const std::vector &, const ImageSentinels &, + size_t dimension, const ImageEncode &, const ImageCancelled &, + ImageRows & output, std::string & error); + +// Only complete image blocks may intersect a chunk. The normal embed callback +// receives validated ordinary token IDs only; mixed output commits atomically. +bool embed_image_prompt_chunk(const PreparedImagePrompt &, const ImageRows &, + int32_t vocabulary, size_t dimension, + size_t position, size_t count, const TextEmbed &, + std::vector & output, std::string & error); + +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_image_budget.h b/server/src/deepseek4/deepseek4_image_budget.h new file mode 100644 index 000000000..5818095bb --- /dev/null +++ b/server/src/deepseek4/deepseek4_image_budget.h @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +namespace luce::vision { + +inline constexpr uint64_t SCRATCH_RESERVATION = 2ULL * 1024 * 1024 * 1024; + +inline uint64_t remaining_expert_budget(uint64_t total, uint64_t core, + uint64_t kv, uint64_t warm, uint64_t safety, + uint64_t vision_reservation, uint64_t already_resident_workspace = 0) { + if (already_resident_workspace > vision_reservation) return 0; + for (uint64_t charge : {core, kv, warm, safety, + vision_reservation - already_resident_workspace}) { + if (charge > total) return 0; + total -= charge; + } + return total; +} + +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_image_policy.cpp b/server/src/deepseek4/deepseek4_image_policy.cpp new file mode 100644 index 000000000..9b650aa78 --- /dev/null +++ b/server/src/deepseek4/deepseek4_image_policy.cpp @@ -0,0 +1,51 @@ +#include "deepseek4_image_policy.h" +#include +#include + +namespace luce::vision { +bool select_image_experts(const float * scores, const float * bias, size_t experts, + size_t topk, ImageExpertSelection & output, std::string & error, + float route_scale) { + output = {}; error.clear(); + if (!scores || !bias || experts==0 || experts>MAX_IMAGE_EXPERTS || topk==0 || topk>experts) { + error="invalid expert arrays/count/topk"; return false; + } + if (!std::isfinite(route_scale) || route_scale<=0) { + error="route scale must be finite and positive"; return false; + } + std::array corrected{}; + std::array order{}; + for (size_t i=0;i(i); + } + std::sort(order.begin(),order.begin()+experts,[&](int32_t a,int32_t b) { + return corrected[a]>corrected[b] || (corrected[a]==corrected[b] && a=image_begin && query=image_begin && key +#include +#include +#include + +namespace luce::vision { + +constexpr size_t MAX_IMAGE_EXPERTS = 256; +struct ImageExpertSelection { + size_t count = 0; + std::array indices{}; + std::array weights{}; +}; + +// Scores are unbiased sqrt(softplus(logits)); bias affects selection only. +// Input arrays contain experts readable floats and do not alias output. +// Equal corrected scores select lower expert indices first. Torch topk does +// not specify equal-score ordering, so tie parity is not part of this contract. +bool select_image_experts(const float * scores, const float * bias, size_t experts, + size_t topk, ImageExpertSelection & output, std::string & error, + float route_scale = 1.5f); + +// Only raw keys are covered. Pass (-1,-1) for no image; otherwise the range is +// [IMAGE_START, IMAGE_END+1), excluding leading compression padding. A range +// widens visibility only when it contains query. False means invalid arguments; +// visible is cleared on failure. No request span or compressed-row state is owned. +bool raw_key_visible(int64_t query, int64_t key, int64_t window, + int64_t image_begin, int64_t image_end, bool & visible); + +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_image_prompt.cpp b/server/src/deepseek4/deepseek4_image_prompt.cpp new file mode 100644 index 000000000..70e02d661 --- /dev/null +++ b/server/src/deepseek4/deepseek4_image_prompt.cpp @@ -0,0 +1,99 @@ +#include "deepseek4_image_prompt.h" +#include +#include +#include + +namespace luce::vision { +namespace { +PreparedImagePrompt fail(ImagePromptError error,const std::string & message) { + PreparedImagePrompt result; + result.error=error; + result.message=message; + return result; +} +} +PreparedImagePrompt prepare_image_prompt(const std::vector & tokens, + const std::vector & images,const ImagePromptLimits & limits, + const ImageTokenizerContract & tokenizer) { + if (tokenizer.vocabulary!=129280 || tokenizer.marker!=129264) + return fail(ImagePromptError::InvalidContract,"unsupported image tokenizer contract"); + if (limits.context_capacity==0 || limits.context_capacity>std::numeric_limits::max() || + limits.output_reserve>limits.context_capacity || limits.max_expanded_tokens==0 || + limits.max_expanded_tokens>MAX_PREPARED_PROMPT_TOKENS) + return fail(ImagePromptError::InvalidLimits,"invalid context, output reserve, or prompt bound"); + if (images.size()>4) return fail(ImagePromptError::ImageCount,"at most four images are supported"); + if (tokens.size()>limits.max_expanded_tokens) + return fail(ImagePromptError::TokenLimit,"rendered tokens exceed prompt bound"); + size_t markers=0; + for (int32_t token:tokens) { + if (token<0 || static_cast(token)>=tokenizer.vocabulary) + return fail(ImagePromptError::InvalidToken,"rendered token outside text vocabulary"); + if (token==tokenizer.marker) ++markers; + } + if (markers!=images.size()) return fail(ImagePromptError::MarkerCount,"final image marker count differs from image count"); + + try { + for (size_t i=0;i0x3f80U) + return fail(ImagePromptError::InvalidPatches,where+"patch value outside finite normalized range"); + } + } + std::vector layouts; + layouts.reserve(images.size()); + uint64_t position=0; + for (int32_t token:tokens) { + uint64_t added=1; + if (token==tokenizer.marker) { + const auto & plan=images[layouts.size()].plan; + ImageLayout layout; + if (!build_image_layout(plan.aligner_rows,plan.aligner_cols,position,layout)) + return fail(ImagePromptError::InvalidPlan,"image layout could not be built"); + added=layout.types.size(); + layouts.push_back(std::move(layout)); + } + if (added>limits.max_expanded_tokens-position) + return fail(ImagePromptError::TokenLimit,"expanded tokens exceed prompt bound"); + position+=added; + } + if (position>limits.context_capacity-limits.output_reserve) + return fail(ImagePromptError::ContextOverflow,"expanded prompt and output reserve exceed context capacity"); + + PreparedImagePrompt result; + result.tokens.reserve(static_cast(position)); + result.images.reserve(images.size()); + size_t image_index=0; + for (int32_t token:tokens) { + if (token!=tokenizer.marker) { result.tokens.push_back(token); continue; } + auto & layout=layouts[image_index]; + for (ImageTokenType kind:layout.types) + result.tokens.push_back(static_cast(tokenizer.vocabulary)+static_cast(kind)); + result.images.push_back({images[image_index],std::move(layout)}); + ++image_index; + } + return result; + } catch (const std::bad_alloc &) { + return fail(ImagePromptError::AllocationFailed,"image prompt allocation failed"); + } +} +} diff --git a/server/src/deepseek4/deepseek4_image_prompt.h b/server/src/deepseek4/deepseek4_image_prompt.h new file mode 100644 index 000000000..4fb115320 --- /dev/null +++ b/server/src/deepseek4/deepseek4_image_prompt.h @@ -0,0 +1,48 @@ +#pragma once +#include "deepseek4_vision_preprocess.h" + +namespace luce::vision { + +struct ImagePatchInput { + ResizePlan plan; + std::vector patches_bf16; +}; +struct PromptImage { + ImagePatchInput input; + ImageLayout layout; +}; +// Text the DS4V chat template renders for one image; it tokenizes to `marker`. +inline constexpr char DS4V_IMAGE_PLACEHOLDER[] = "<｜deepseek_image｜>"; + +struct ImageTokenizerContract { + std::uint32_t vocabulary = 129280; + std::int32_t marker = 129264; +}; +struct ImagePromptLimits { + std::uint64_t context_capacity = 131072; + std::uint64_t output_reserve = 4096; + std::uint64_t max_expanded_tokens = 131072; +}; +constexpr std::uint64_t MAX_PREPARED_PROMPT_TOKENS = 1048576; +enum class ImagePromptError { + None, InvalidContract, InvalidLimits, InvalidToken, ImageCount, + MarkerCount, InvalidPlan, InvalidPatches, TokenLimit, ContextOverflow, AllocationFailed, +}; +struct PreparedImagePrompt { + ImagePromptError error = ImagePromptError::None; + std::string message; + std::vector tokens; + std::vector images; + explicit operator bool() const { return error == ImagePromptError::None; } +}; + +// Consumes final rendered token IDs and already-preprocessed image inputs. +// Success owns independent copies of patches and layouts. Failure owns no +// partial tokens/images. No decoding, model execution, or cache policy occurs. +PreparedImagePrompt prepare_image_prompt( + const std::vector & rendered_tokens, + const std::vector & images, + const ImagePromptLimits & limits = {}, + const ImageTokenizerContract & tokenizer = {}); + +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_image_spans.h b/server/src/deepseek4/deepseek4_image_spans.h new file mode 100644 index 000000000..3b4d1ed6e --- /dev/null +++ b/server/src/deepseek4/deepseek4_image_spans.h @@ -0,0 +1,15 @@ +// DS4V limits for the shared image span helpers. +#pragma once + +#include "../common/vision/image_spans.h" + +namespace luce::vision { + +inline constexpr size_t DS4V_MAX_IMAGES = 4; +inline constexpr uint64_t DS4V_MAX_IMAGE_BLOCK_TOKENS = 384; + +inline bool valid_image_spans(ImageSpanView spans, uint64_t prompt_size) { + return valid_image_spans(spans, prompt_size, DS4V_MAX_IMAGES, DS4V_MAX_IMAGE_BLOCK_TOKENS); +} + +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_internal.h b/server/src/deepseek4/deepseek4_internal.h index ff4de1dc9..5872ff703 100644 --- a/server/src/deepseek4/deepseek4_internal.h +++ b/server/src/deepseek4/deepseek4_internal.h @@ -27,6 +27,7 @@ #include "common/layer_split_utils.h" #include "common/paged_attention_config.h" #include "common/prefill_attention_mode.h" +#include "deepseek4_image_spans.h" #include "common/concurrency/paged_kv_pool.h" #include "deepseek4_paged_cache.h" @@ -138,6 +139,7 @@ struct DeepSeek4Layer { // Router ggml_tensor * ffn_gate_inp = nullptr; // [n_embd, n_expert] router weights F16 ggml_tensor * ffn_exp_probs_b = nullptr; // [n_expert] optional routing bias + ggml_tensor * ffn_gate_bias_vl = nullptr; // image router bias, loaded only with --mmproj // Hash routing table (first n_hash_layer layers only) ggml_tensor * ffn_gate_tid2eid = nullptr; // [n_expert_used, n_vocab] I32 @@ -247,6 +249,12 @@ struct DeepSeek4Weights { bool fused_verify_f16_kv = false; }; +// True when the image router biases were loaded, i.e. the backend was started +// with a vision projector. +inline bool ds4_image_capable(const DeepSeek4Weights & w) { + return !w.layers.empty() && w.layers.front().ffn_gate_bias_vl != nullptr; +} + inline bool deepseek4_is_eos_tok(int tok, const DeepSeek4Weights & w) { return (w.eos_chat_id >= 0 && tok == w.eos_chat_id) || (w.eos_id >= 0 && tok == w.eos_id); @@ -360,6 +368,7 @@ struct DeepSeek4Head4Tail2Routes { struct DeepSeek4BackendConfig { std::string model_path; + std::string mmproj_path; DevicePlacement device; int stream_fd = -1; int chunk = 512; // prefill chunk size @@ -467,6 +476,10 @@ void reset_deepseek4_cache(DeepSeek4Cache & c); // state and the DSpark feature tail remain live for the following decode. void deepseek4_release_prefill_scratch(DeepSeek4Cache & c, MoeHybridStorage * moe_hybrid); +// Retire all disposable decoder/owner graphs before the vision tower uses the +// shared scratch allowance. KV and saved snapshots are left intact. +void deepseek4_release_image_scratch(DeepSeek4Cache & c, + MoeHybridStorage * moe_hybrid); // Invalid/future raw-ring rows after all writes of a batched verifier. // Each span is bounded by n_swa, including batches that overwrite the full ring. int deepseek4_verify_raw_mask_spans( @@ -548,7 +561,14 @@ bool deepseek4_step_layer_range( Ds4VerifyHooks * verify_hooks = nullptr, MoeHybridStorage * moe_hybrid = nullptr, MoeExpertComputeRuntime * expert_runtime = nullptr, - MoeHybridRoutingStats * routing_stats = nullptr); + MoeHybridRoutingStats * routing_stats = nullptr, + vision::ImageSpanView image_spans = {}); + +bool deepseek4_validate_image_batch( + const DeepSeek4Weights & w, const DeepSeek4Cache & cache, + const MoeHybridStorage * hybrid, const int32_t * tokens, + int count, int position, vision::ImageSpanView spans, + bool & has_images, std::string & error); bool build_deepseek4_moe_hybrid_storage_from_file( const std::string & path, diff --git a/server/src/deepseek4/deepseek4_loader.cpp b/server/src/deepseek4/deepseek4_loader.cpp index 2169c9c41..82d227e82 100644 --- a/server/src/deepseek4/deepseek4_loader.cpp +++ b/server/src/deepseek4/deepseek4_loader.cpp @@ -18,6 +18,8 @@ #include "luce.h" #include "common/gguf_bounds.h" #include "../common/moe_hybrid_storage.h" +#include "../common/copied_source_reclaim.h" +#include "../common/copied_source_upload.h" #include "../common/moe_hybrid_types.h" #include "ggml-cuda.h" @@ -27,6 +29,7 @@ #include // SIZE_MAX, used by the portable checked-size helpers below #include #include +#include #include #include #include @@ -234,6 +237,27 @@ static bool is_expert_tensor(const char * name) { std::strstr(name, "ffn_down_exps") != nullptr; } +// Layer index of an image router bias, or -1. Two spellings exist: the source +// checkpoint's "layers.N.ffn.gate.bias_vl" (kept by our converter) and +// llama.cpp's "blk.N.exp_probs_b_vl.bias" (published GGUFs). Leading zeros and +// trailing text are rejected. +static int image_bias_layer(const char * name) { + for (const auto & [prefix, suffix] : {std::pair + {"layers.", ".ffn.gate.bias_vl"}, {"blk.", ".exp_probs_b_vl.bias"}}) { + const size_t prefix_len = std::strlen(prefix); + if (std::strncmp(name, prefix, prefix_len) != 0) continue; + const char * number = name + prefix_len; + if (*number < '0' || *number > '9') return -1; + char * rest = nullptr; + const long layer = std::strtol(number, &rest, 10); + if (layer < 0 || layer > std::numeric_limits::max() || + std::strcmp(rest, suffix) != 0 || + std::string(name) != prefix + std::to_string(layer) + suffix) return -1; + return int(layer); + } + return -1; +} + static bool should_keep_ds4_tensor(const char * name, const TargetLoadPlan & plan) { int layer_id = -1; @@ -244,6 +268,12 @@ static bool should_keep_ds4_tensor(const char * name, is_expert_tensor(name); } + const int image_layer = image_bias_layer(name); + if (image_layer >= 0) { + return plan.load_ds4_image_bias && image_layer >= plan.layer_begin && + image_layer < plan.layer_end; + } + // Global tensors if (std::strcmp(name, "token_embd.weight") == 0 || std::strcmp(name, "output_norm.weight") == 0 || @@ -1423,7 +1453,6 @@ bool load_deepseek4_gguf_partial(const std::string & path, static const char * kRequiredU32Keys[] = { "deepseek4.block_count", "deepseek4.embedding_length", - "deepseek4.vocab_size", "deepseek4.attention.head_count", "deepseek4.attention.head_count_kv", "deepseek4.attention.key_length", @@ -1455,7 +1484,15 @@ bool load_deepseek4_gguf_partial(const std::string & path, // ── Read hyperparameters ──────────────────────────────────────────── const uint32_t n_layer = get_u32_or(gctx, "deepseek4.block_count", 43); const uint32_t n_embd = get_u32_or(gctx, "deepseek4.embedding_length", 4096); - const uint32_t n_vocab = get_u32_or(gctx, "deepseek4.vocab_size", 129280); + // llama.cpp conversions carry no vocab_size key; the token list has the size. + uint32_t n_vocab = get_u32_or(gctx, "deepseek4.vocab_size", 0); + if (n_vocab == 0) { + const int64_t tokens_key = gguf_find_key(gctx, "tokenizer.ggml.tokens"); + if (tokens_key >= 0 && gguf_get_kv_type(gctx, tokens_key) == GGUF_TYPE_ARRAY) { + const int64_t n_tokens = (int64_t) gguf_get_arr_n(gctx, tokens_key); + if (n_tokens > 0 && n_tokens <= std::numeric_limits::max()) n_vocab = (uint32_t) n_tokens; + } + } const uint32_t n_head = get_u32_or(gctx, "deepseek4.attention.head_count", 64); const uint32_t n_head_kv = get_u32_or(gctx, "deepseek4.attention.head_count_kv", 1); const uint32_t head_dim = get_u32_or(gctx, "deepseek4.attention.key_length", 512); @@ -1490,7 +1527,7 @@ bool load_deepseek4_gguf_partial(const std::string & path, const float swiglu_clamp = get_f32_or(gctx, "deepseek4.swiglu_clamp_exp", 10.0f); if (n_vocab == 0) { - set_last_error("deepseek4.vocab_size must be > 0"); + set_last_error("no vocabulary size: need deepseek4.vocab_size or tokenizer.ggml.tokens"); gguf_free(gctx); if (meta_ctx) ggml_free(meta_ctx); return false; @@ -1574,6 +1611,32 @@ bool load_deepseek4_gguf_partial(const std::string & path, // ── Collect tensors for allocation ────────────────────────────────── const int n_tensors = gguf_get_n_tensors(gctx); + if (plan.load_ds4_image_bias && !plan.expert_metadata_only) { + // Image routing needs one F32[n_expert] router bias per decoder layer. + bool valid = plan.layer_begin == 0 && plan.layer_end == int(n_layer); + std::vector counts(size_t(n_layer), 0); + for (int ti = 0; ti < n_tensors; ++ti) { + const char * name = gguf_get_tensor_name(gctx, ti); + const int layer = image_bias_layer(name); + if (layer < 0) continue; + if (layer >= int(n_layer)) continue; // the MTP block's bias is not loaded + ++counts[size_t(layer)]; + const ggml_tensor * tensor = find_tensor(meta_ctx, name); + valid = valid && tensor && tensor->type == GGML_TYPE_F32 && + tensor->ne[0] == int64_t(n_expert) && tensor->ne[1] == 1 && + tensor->ne[2] == 1 && tensor->ne[3] == 1; + } + valid = valid && std::all_of(counts.begin(), counts.end(), [](int count) { return count == 1; }); + if (!valid) { + set_last_error("--mmproj requires a full-range load and one F32[n_expert] image router bias per layer"); + gguf_free(gctx); + if (meta_ctx) ggml_free(meta_ctx); + return false; + } + } + // A vision load packs the primary GPU tightly: hand file-backed source + // pages back to the kernel as soon as each tensor has been copied. + const bool reclaim_sources = plan.load_ds4_image_bias; const size_t data_offset = gguf_get_data_offset(gctx); ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backend); const size_t alignment = ggml_backend_buft_get_alignment(buft); @@ -1777,6 +1840,13 @@ bool load_deepseek4_gguf_partial(const std::string & path, if (!a.upload_to_backend || !a.dense_split) continue; const void * src_data = (const char *)mmap.addr + a.file_offset; ggml_backend_tensor_set(a.tensor, src_data, 0, a.file_size); +#if defined(__linux__) + // set_tensor has completed its source copy, including split buffers. + if (reclaim_sources) { + reclaim_copied_file_source(mmap.addr, mmap.len, src_data, a.file_size, + mmap.fd, ggml_get_name(a.tensor)); + } +#endif } if (!read_ok) { set_last_error("parallel weight read failed"); @@ -1788,10 +1858,42 @@ bool load_deepseek4_gguf_partial(const std::string & path, return false; } } else { +#if defined(__linux__) && (defined(LUCE_BACKEND_HIP) || defined(GGML_USE_HIP)) + std::vector upload_scratch; +#endif for (auto & a : allocs) { if (!a.upload_to_backend) continue; const void * src_data = (const char *)mmap.addr + a.file_offset; - ggml_backend_tensor_set(a.tensor, src_data, 0, a.file_size); +#if defined(__linux__) && (defined(LUCE_BACKEND_HIP) || defined(GGML_USE_HIP)) + if (reclaim_sources && !a.dense_split && ggml_backend_is_cuda(backend) && + !ggml_backend_cuda_buffer_is_managed(buf)) { + // HIP may pin pageable upload sources. Keep file-backed pages + // out of that path so completed-source cache advice can act. + if (!upload_copied_file_chunks(mmap.addr, mmap.len, a.file_offset, + a.file_size, upload_scratch, + [&](const uint8_t * bytes, size_t offset, size_t count) { + ggml_backend_tensor_set(a.tensor, bytes, offset, count); + })) { + set_last_error("invalid dense staged-upload source range"); + mmap.close_map(); + if (split_buf) ggml_backend_buffer_free(split_buf); + if (buf) ggml_backend_buffer_free(buf); + gguf_free(gctx); + ggml_free(meta_ctx); + return false; + } + } else +#endif + { + ggml_backend_tensor_set(a.tensor, src_data, 0, a.file_size); + } +#if defined(__linux__) + // set_tensor has completed its source copy, including split buffers. + if (reclaim_sources) { + reclaim_copied_file_source(mmap.addr, mmap.len, src_data, a.file_size, + mmap.fd, ggml_get_name(a.tensor)); + } +#endif } } mmap.close_map(); @@ -1809,6 +1911,13 @@ bool load_deepseek4_gguf_partial(const std::string & path, if (emb_mmap.open_ro(path, emb_err)) { std::memcpy(out.embedder.tok_embd_owned.data(), (const char *)emb_mmap.addr + a.file_offset, a.file_size); +#if defined(__linux__) + if (reclaim_sources) { + reclaim_copied_file_source(emb_mmap.addr, emb_mmap.len, + (const char *)emb_mmap.addr + a.file_offset, a.file_size, + emb_mmap.fd, "token_embd.weight"); + } +#endif emb_mmap.close_map(); } else { set_last_error("embedder mmap: " + emb_err); @@ -1829,6 +1938,12 @@ bool load_deepseek4_gguf_partial(const std::string & path, for (auto & a : allocs) { const char * name = ggml_get_name(a.tensor); + const int image_layer = image_bias_layer(name); + if (plan.load_ds4_image_bias && image_layer >= 0 && image_layer < int(n_layer)) { + out.layers[size_t(image_layer)].ffn_gate_bias_vl = a.tensor; + continue; + } + // Global tensors if (std::strcmp(name, "token_embd.weight") == 0) { out.tok_embd = a.tensor; continue; } if (std::strcmp(name, "output_norm.weight") == 0) { out.out_norm = a.tensor; continue; } @@ -2091,7 +2206,9 @@ bool build_deepseek4_moe_hybrid_storage_from_file_with_mmap( if (err) *err = mmap_err; return false; } +#if !defined(__linux__) mmap.close_fd(); +#endif const size_t data_start = gguf_get_data_offset(gctx); const auto * file_bytes = static_cast(mmap.addr); @@ -2138,7 +2255,13 @@ bool build_deepseek4_moe_hybrid_storage_from_file_with_mmap( const MoeHybridConfig cfg = cfg_override ? *cfg_override : make_ds4_moe_hybrid_config(w); const bool ok = build_moe_hybrid_storage_from_file_with_mmap( cfg, backend, placement, layer_descs, layer_file_data, - mmap.addr, mmap.len, out, err, 0, cold_gpu_backend); + mmap.addr, mmap.len, out, err, 0, cold_gpu_backend +#if defined(__linux__) + , ds4_image_capable(w) ? mmap.fd : -1 +#endif + ); + // Advice borrows the original fd only while construction is in progress. + mmap.close_fd(); if (!ok) { mmap.close_map(); diff --git a/server/src/deepseek4/deepseek4_norm.h b/server/src/deepseek4/deepseek4_norm.h new file mode 100644 index 000000000..677c01200 --- /dev/null +++ b/server/src/deepseek4/deepseek4_norm.h @@ -0,0 +1,13 @@ +#pragma once +#include "ggml.h" + +namespace luce::common::detail { +inline ggml_tensor * build_rms_norm(ggml_context * ctx, ggml_tensor * x, + ggml_tensor * weight, float eps) { + // HIP binary broadcast does not accept BF16 affine operands. Widen only + // this small vector; stored weights and matrix products remain unchanged. + if (weight->type == GGML_TYPE_BF16) weight = ggml_cast(ctx, weight, GGML_TYPE_F32); + ggml_tensor * normed = ggml_rms_norm(ctx, x, eps); + return ggml_mul(ctx, normed, weight); +} +} diff --git a/server/src/deepseek4/deepseek4_vision.cpp b/server/src/deepseek4/deepseek4_vision.cpp new file mode 100644 index 000000000..593dab6ee --- /dev/null +++ b/server/src/deepseek4/deepseek4_vision.cpp @@ -0,0 +1,475 @@ +#include "deepseek4_vision.h" +#include "common/gguf_bounds.h" +#include "common/gguf_mmap.h" +#include "ggml.h" +#include "ggml-alloc.h" +#include "gguf.h" +#include +#include +#include +#include +#include +#include + +namespace luce::vision { +namespace { +using Tensor = ggml_tensor; +constexpr size_t MAX_SCRATCH = size_t(2) * 1024 * 1024 * 1024; +Tensor * rounded(ggml_context * c, Tensor * x) { + return ggml_cast(c, ggml_cast(c, x, GGML_TYPE_BF16), GGML_TYPE_F32); +} +void require(bool value, const std::string & error) { if (!value) throw std::runtime_error(error); } +struct Meta { + gguf_context * g = nullptr; + ggml_context * c = nullptr; + ~Meta() { if (g) gguf_free(g); if (c) ggml_free(c); } +}; +std::map> inventory() { + std::map> result; + auto add = [&](const std::string & n, int64_t a, int64_t b = 1) { result[n] = {a,b,1,1}; }; + add("vision.patch_embed.proj.weight",588,1024); add("vision.patch_embed.proj.bias",1024); + add("vision.norm.weight",1024); + add("aligner.w1.weight",9216,4096); add("aligner.w1.bias",4096); + add("aligner.w2.weight",4096,4096); add("aligner.w2.bias",4096); + for (auto n : {"image_start","image_pad","image_newline","image_end"}) add(n,4096); + for (int i=0;i<32;++i) { + const auto p = "vision.blocks." + std::to_string(i); + add(p+".norm1.weight",1024); add(p+".norm2.weight",1024); + add(p+".attn.wqkv.weight",1024,3072); add(p+".attn.wqkv.bias",3072); + add(p+".attn.wo.weight",1024,1024); add(p+".attn.wo.bias",1024); + add(p+".mlp.w1.weight",1024,5632); add(p+".mlp.w2.weight",2816,1024); + } + return result; +} +void validate_metadata(gguf_context * g, int dimension, int vocabulary) { + auto key = [&](const std::string & n, gguf_type type) { + auto id = gguf_find_key(g,n.c_str()); + require(id >= 0 && gguf_get_kv_type(g,id)==type,"missing or wrong metadata type: "+n); + return id; + }; + auto str = [&](const std::string & n, const char * value) { + require(std::string(gguf_get_val_str(g,key(n,GGUF_TYPE_STRING)))==value,"unsupported metadata: "+n); + }; + auto integer = [&](const std::string & n, uint32_t value) { + require(gguf_get_val_u32(g,key(n,GGUF_TYPE_UINT32))==value,"unsupported metadata: "+n); + }; + auto real = [&](const std::string & n,float value) { + require(gguf_get_val_f32(g,key(n,GGUF_TYPE_FLOAT32))==value,"unsupported metadata: "+n); + }; + str("general.architecture","deepseek4_vision"); str("general.type","mmproj"); + integer("general.alignment",32); + const std::string p="deepseek4.vision."; + for (auto entry : std::map{{"schema_version",1},{"block_count",32}, + {"embedding_length",1024},{"attention.head_count",16},{"attention.head_dimension",64}, + {"feed_forward_length",2816},{"patch_size",14},{"downsample_ratio",3},{"aligner_input_length",9216}, + {"language_embedding_length",4096},{"vocabulary_size",129280},{"image.max_tokens",384}, + {"image.min_pixels",147456},{"image.layout_version",1},{"image.compression_alignment",4}, + {"image.sentinel_type_count",5}}) integer(p+entry.first,entry.second); + require(dimension==4096 && vocabulary==129280,"language model dimension/vocabulary mismatch"); + real(p+"rope.freq_base",10000.f); real(p+"attention.layer_norm_rms_epsilon",1e-6f); + real(p+"image.max_aspect_ratio",8.f); + for (auto entry : std::map{{"attention.rope_layout","2d-half-split-height-width"}, + {"image.patch_layout","channel-major"},{"image.layout","n"}, + {"image.layout_recipe","row-pair-column-interleave"},{"image.sentinel_types","start,pad,image,newline,end"}, + {"aligner.padding","bottom-right"},{"aligner.patch_layout","channel-first-unfold"}, + {"aligner.activation","gelu-exact"}}) str(p+entry.first,entry.second); + for (auto n : {"image.normalization_mean","image.normalization_std"}) { + auto id = key(p+n,GGUF_TYPE_ARRAY); + require(gguf_get_arr_type(g,id)==GGUF_TYPE_FLOAT32 && gguf_get_arr_n(g,id)==3,"bad normalization array"); + const auto * values = static_cast(gguf_get_arr_data(g,id)); + require(values[0]==.5f && values[1]==.5f && values[2]==.5f,"unsupported normalization"); + } +} +struct Graph { + ggml_context * c; + ggml_cgraph * graph; + std::vector> stages; + explicit Graph() { + const size_t nodes=512; + c=ggml_init({nodes*ggml_tensor_overhead()+ggml_graph_overhead_custom(nodes,false),nullptr,true}); + require(c!=nullptr,"graph metadata allocation failed"); + graph=ggml_new_graph_custom(c,nodes,false); + } + ~Graph() { ggml_free(c); } + void stage(const std::string & name,Tensor * t,bool enabled) { + if (enabled) { + // An output flag on a view does not pin its underlying allocation. + // A dedicated snapshot prevents later in-place operations or allocator + // reuse from corrupting diagnostic values (notably reshape/unfold). + auto snapshot=ggml_dup(c,t); + ggml_set_output(snapshot); stages.emplace_back(name,snapshot); + } + } +}; +std::vector read(Tensor * t) { + std::vector result(ggml_nelements(t)); + ggml_backend_tensor_get(t,result.data(),0,result.size()*sizeof(float)); + return result; +} +} +namespace detail { +static size_t hip_size_query(ggml_backend_t backend,const char * name) { + if(!backend) return 0; + auto device=ggml_backend_get_device(backend); + if(!device) return 0; + auto reg=ggml_backend_dev_backend_reg(device); + if(!reg) return 0; + using Query=size_t (*)(ggml_backend_t); + auto query=reinterpret_cast(ggml_backend_reg_get_proc_address(reg,name)); + return query ? query(backend) : 0; +} +size_t hip_bias_workspace(ggml_backend_t b) { return hip_size_query(b,"ggml_backend_hip_vision_bias_bf16_workspace"); } +size_t hip_bias_launches(ggml_backend_t b) { return hip_size_query(b,"ggml_backend_hip_vision_bias_bf16_launches"); } +size_t hip_norm_launches(ggml_backend_t b) { return hip_size_query(b,"ggml_backend_hip_vision_norm_f32_launches"); } +size_t hip_rotary_launches(ggml_backend_t b) { return hip_size_query(b,"ggml_backend_hip_vision_rotary_f32_launches"); } +size_t hip_softmax_launches(ggml_backend_t b) { return hip_size_query(b,"ggml_backend_hip_vision_softmax_f32_launches"); } +size_t hip_av_launches(ggml_backend_t b) { return hip_size_query(b,"ggml_backend_hip_vision_av_f32_launches"); } +bool hip_softmax_capable(ggml_backend_t backend) { + if(!backend) return false; + auto device=ggml_backend_get_device(backend); + if(!device) return false; + auto reg=ggml_backend_dev_backend_reg(device); + if(!reg) return false; + using Query=bool (*)(ggml_backend_t); + auto query=reinterpret_cast(ggml_backend_reg_get_proc_address(reg,"ggml_backend_hip_vision_softmax_f32_capable")); + return query && query(backend); +} +bool hip_av_capable(ggml_backend_t backend) { + if(!backend) return false; + auto device=ggml_backend_get_device(backend); + if(!device) return false; + auto reg=ggml_backend_dev_backend_reg(device); + if(!reg) return false; + using Query=bool (*)(ggml_backend_t); + auto query=reinterpret_cast(ggml_backend_reg_get_proc_address(reg,"ggml_backend_hip_vision_av_f32_capable")); + return query && query(backend); +} +bool hip_rotary_capable(ggml_backend_t backend) { + if(!backend) return false; + auto device=ggml_backend_get_device(backend); + if(!device) return false; + auto reg=ggml_backend_dev_backend_reg(device); + if(!reg) return false; + using Query=bool (*)(ggml_backend_t); + auto query=reinterpret_cast(ggml_backend_reg_get_proc_address(reg,"ggml_backend_hip_vision_rotary_f32_capable")); + return query && query(backend); +} +bool hip_norm_capable(ggml_backend_t backend) { + if(!backend) return false; + auto device=ggml_backend_get_device(backend); + if(!device) return false; + auto reg=ggml_backend_dev_backend_reg(device); + if(!reg) return false; + using Query=bool (*)(ggml_backend_t); + auto query=reinterpret_cast(ggml_backend_reg_get_proc_address(reg,"ggml_backend_hip_vision_norm_f32_capable")); + return query && query(backend); +} +Tensor * rms_norm(ggml_context * c,Tensor * input,float epsilon,ggml_backend_t backend) { + if(!hip_bias_workspace(backend)) return ggml_rms_norm(c,input,epsilon); + require(hip_norm_capable(backend),"HIP source-order vision normalization unavailable; fallback forbidden"); + auto y=ggml_rms_norm_vision_f32(c,input,epsilon); + require(ggml_backend_supports_op(backend,y),"HIP source-order vision normalization unsupported; fallback forbidden"); + return y; +} +Tensor * linear(ggml_context * c,Tensor * weight,Tensor * input,Tensor * bias,bool preserve_biased_product,ggml_backend_t backend) { + // HIP's explicit capability is required: generic supports_op defaults on + // other backends are not evidence of this source-specific operation. + if(hip_bias_workspace(backend)) { + auto y=ggml_mul_mat_bias_bf16(c,weight,ggml_cast(c,input,GGML_TYPE_BF16),bias); + require(ggml_backend_supports_op(backend,y),"HIP BF16 vision linear unsupported; fallback forbidden"); + return ggml_cast(c,y,GGML_TYPE_F32); + } + // Source biased linears round after the bias. The HIP BF16 BLAS path can + // round its product to BF16 even with GGML_PREC_F32; an F32 weight operand + // preserves the product until the explicit final boundary below. + // The CPU path already retains an F32 product; keep its existing reduction. + if(bias && preserve_biased_product) weight=ggml_cast(c,weight,GGML_TYPE_F32); + auto y=ggml_mul_mat(c,weight,input); + ggml_mul_mat_set_prec(y,GGML_PREC_F32); + if(bias) y=ggml_add(c,y,ggml_cast(c,bias,GGML_TYPE_F32)); + return rounded(c,y); +} +void rotary_tables(PatchGrid grid,std::vector & cosine,std::vector & sine,ggml_backend_t backend) { + require(grid.height>0 && grid.width>0 && grid.height<=1152 && grid.width<=1152,"invalid rotary grid"); + const int n=grid.height*grid.width; + if(hip_bias_workspace(backend)) { + const size_t temporary=(32+size_t(n)*32*3)*sizeof(float); + constexpr size_t limit=128ULL*1024*1024; + const size_t external=hip_bias_workspace(backend); + require(external<=limit && temporary<=limit-external,"HIP rotary temporary storage exceeds 128 MiB bound"); + require(hip_rotary_capable(backend),"HIP source-order vision rotary tables unavailable; fallback forbidden"); + using Fill=bool (*)(ggml_backend_t,int64_t,int64_t,float *,float *); + auto reg=ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)); + auto fill=reinterpret_cast(ggml_backend_reg_get_proc_address(reg,"ggml_backend_hip_vision_rotary_f32")); + require(fill!=nullptr,"HIP source-order vision rotary table function unavailable"); + cosine.resize(size_t(n)*32); sine.resize(size_t(n)*32); + require(fill(backend,grid.height,grid.width,cosine.data(),sine.data()),"HIP source-order vision rotary table preparation failed"); + return; + } + cosine.resize(size_t(n)*32); sine.resize(size_t(n)*32); + for(int i=0;ine[1],x->ne[2],x->nb[1],x->nb[2],0); + auto b=ggml_view_3d(c,x,32,x->ne[1],x->ne[2],x->nb[1],x->nb[2],32*sizeof(float)); + auto first=ggml_sub(c,ggml_mul(c,a,cosine),ggml_mul(c,b,sine)); + auto second=ggml_add(c,ggml_mul(c,b,cosine),ggml_mul(c,a,sine)); + return rounded(c,ggml_concat(c,first,second,0)); +} +Tensor * attention(ggml_context * c,Tensor * q,Tensor * k,Tensor * v,ggml_backend_t backend) { + q=ggml_cont(c,ggml_permute(c,q,0,2,1,3)); + k=ggml_cont(c,ggml_permute(c,k,0,2,1,3)); + // PyTorch 2.10 Math SDPA (the source's 3D input dispatch) scales both + // operands in F32 before GEMM. Preserve this order and its rounding; + // scaling the scores afterwards is algebraically equal but not bit equal. + const float scale=float(std::sqrt(1.0/std::sqrt(double(q->ne[0])))); + auto scores=ggml_mul_mat(c,ggml_scale(c,k,scale),ggml_scale(c,q,scale)); + ggml_mul_mat_set_prec(scores,GGML_PREC_F32); + Tensor * out; + if(hip_bias_workspace(backend)) { + require(hip_softmax_capable(backend) && hip_av_capable(backend), + "HIP source-order vision attention unavailable; fallback forbidden"); + auto probabilities=ggml_soft_max_vision_f32(c,scores); + require(ggml_backend_supports_op(backend,probabilities),"HIP vision softmax unsupported"); + // Retain V's token-major layout for the original batched Lt NN call. + out=ggml_mul_mat_vision_av_f32(c,v,probabilities); + require(ggml_backend_supports_op(backend,out),"HIP vision attention product unsupported"); + } else { + auto probabilities=ggml_soft_max(c,scores); + v=ggml_cont(c,ggml_permute(c,v,1,2,0,3)); // [N, D, heads] + out=ggml_mul_mat(c,v,probabilities); + ggml_mul_mat_set_prec(out,GGML_PREC_F32); + } + out=ggml_cont(c,ggml_permute(c,out,0,2,1,3)); + return rounded(c,ggml_reshape_2d(c,out,out->ne[0]*out->ne[1],out->ne[2])); +} +Tensor * unfold(ggml_context * c,Tensor * x,PatchGrid grid,int channels) { + x=ggml_reshape_3d(c,x,channels,grid.width,grid.height); + x=ggml_cont(c,ggml_permute(c,x,2,0,1,3)); // [width,height,channels] + x=ggml_pad(c,x,(3-grid.width%3)%3,(3-grid.height%3)%3,0,0); + auto kernel=ggml_new_tensor_4d(c,GGML_TYPE_F32,3,3,channels,1); + auto out=ggml_im2col(c,kernel,x,3,3,0,0,1,1,true,GGML_TYPE_F32); + return ggml_reshape_2d(c,out,channels*9,((grid.width+2)/3)*((grid.height+2)/3)); +} +} +struct VisionRuntime::Impl { + VisionConfig config; + ggml_backend_t backend=nullptr; + ggml_context * weights=nullptr; + ggml_backend_buffer_t buffer=nullptr; + ggml_gallocr_t allocator=nullptr; + ~Impl() { + if(allocator) ggml_gallocr_free(allocator); + if(buffer) ggml_backend_buffer_free(buffer); + if(weights) ggml_free(weights); + } + Tensor * weight(const std::string & name) { return ggml_get_tensor(weights,name.c_str()); } + Tensor * linear(ggml_context * c,Tensor * x,const std::string & name,bool bias=true) { + const auto device=ggml_backend_get_device(backend); + const bool preserve=device && (ggml_backend_dev_type(device)==GGML_BACKEND_DEVICE_TYPE_GPU || + ggml_backend_dev_type(device)==GGML_BACKEND_DEVICE_TYPE_IGPU); + return detail::linear(c,weight(name+".weight"),x,bias ? weight(name+".bias") : nullptr,preserve,backend); + } + Tensor * norm(ggml_context * c,Tensor * x,const std::string & name) { + return rounded(c,ggml_mul(c,detail::rms_norm(c,x,config.rms_epsilon,backend), + ggml_cast(c,weight(name+".weight"),GGML_TYPE_F32))); + } + std::vector execute(Graph & g,Tensor * input,const std::vector & values,Tensor * output, + const StageObserver & observer,Tensor * cosine=nullptr,Tensor * sine=nullptr, + const std::vector & cos_values={},const std::vector & sin_values={}) { + ggml_set_input(input); ggml_set_output(output); + for(auto & stage:g.stages) ggml_build_forward_expand(g.graph,stage.second); + ggml_build_forward_expand(g.graph,output); + if(!allocator) allocator=ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + require(allocator!=nullptr,"scratch allocator creation failed"); + // reserve_n_size calculates requirements without allocating a backend buffer. + size_t required=0; + ggml_gallocr_reserve_n_size(allocator,g.graph,nullptr,nullptr,&required); + const size_t external=detail::hip_bias_workspace(backend); + require(external<=MAX_SCRATCH && required<=MAX_SCRATCH-external,"vision scratch including retained Lt workspace exceeds 2 GiB bound"); + for(int i=0;ine[0],t->ne[1],t->ne[2],t->ne[3]},read(t)); + } + return read(output); + } +}; +VisionRuntime::VisionRuntime()=default; +VisionRuntime::~VisionRuntime()=default; +bool VisionRuntime::load(const std::string & path,ggml_backend_t backend,int dimension,int vocabulary,std::string & error) { + error.clear(); + try { + require(backend!=nullptr,"null vision backend"); + require(!detail::hip_bias_workspace(backend) || detail::hip_norm_capable(backend), + "HIP source-order vision normalization unavailable"); + require(!detail::hip_bias_workspace(backend) || detail::hip_rotary_capable(backend), + "HIP source-order vision rotary tables unavailable"); + require(!detail::hip_bias_workspace(backend) || (detail::hip_softmax_capable(backend) && detail::hip_av_capable(backend)), + "HIP source-order vision attention unavailable"); + Meta meta; + meta.g=gguf_init_from_file(path.c_str(),{true,&meta.c}); + require(meta.g && meta.c,"could not parse vision GGUF"); + validate_metadata(meta.g,dimension,vocabulary); + auto expected=inventory(); + require(gguf_get_n_tensors(meta.g)==int64_t(expected.size()),"wrong projector tensor count"); + common::GgufMmap mapped; + if(!mapped.open(path,error)) throw std::runtime_error(error); + std::vector> ranges; + for(int64_t i=0;itype==GGML_TYPE_BF16,"wrong tensor dtype: "+name); + for(int d=0;d<4;++d) require(t->ne[d]==it->second[d],"wrong tensor shape: "+name); + const size_t offset=gguf_get_tensor_offset(meta.g,i),size=gguf_get_tensor_size(meta.g,i); + require(common::gguf_tensor_in_file(gguf_get_data_offset(meta.g),offset,size,mapped.size()),"tensor outside file: "+name); + require(offset%32==0,"unaligned tensor: "+name); + ranges.emplace_back(offset,offset+size); + expected.erase(it); + } + std::sort(ranges.begin(),ranges.end()); + for(size_t i=1;i=ranges[i-1].second,"overlapping tensor data"); + require(expected.empty(),"missing projector tensor"); + auto candidate=std::make_unique(); + candidate->backend=backend; + candidate->weights=meta.c; meta.c=nullptr; + candidate->buffer=ggml_backend_alloc_ctx_tensors(candidate->weights,backend); + require(candidate->buffer!=nullptr,"projector weight allocation failed"); + ggml_backend_buffer_set_usage(candidate->buffer,GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + const auto * bytes=static_cast(mapped.data()); + for(int64_t i=0;iweight(gguf_get_tensor_name(meta.g,i)); + ggml_backend_tensor_set(t,bytes+gguf_get_data_offset(meta.g)+gguf_get_tensor_offset(meta.g,i),0,ggml_nbytes(t)); + } + impl_=std::move(candidate); // Failed reload leaves prior runtime intact. + return true; + } catch(const std::exception & e) { error=e.what(); return false; } +} +bool VisionRuntime::encode(const std::vector & patches,PatchGrid grid,VisionOutput & output, + std::string & error,bool retain,const StageObserver & observer) { + error.clear(); output={}; + try { + require(bool(impl_),"vision runtime is not loaded"); + require(grid.height>0 && grid.width>0 && grid.height<=1152 && grid.width<=1152,"invalid patch grid"); + const int64_t n=int64_t(grid.height)*grid.width; + const int64_t aligner_height=(grid.height+2)/3, aligner_width=(grid.width+2)/3; + const int64_t rows=aligner_height*aligner_width; + const int64_t padded_height=aligner_height+aligner_height%2; + const int64_t row_length=aligner_width+1; + const int64_t block_tokens=padded_height*row_length+2+(padded_height/2*row_length%2)*2; + require(block_tokens+3<=impl_->config.max_image_tokens,"patch grid exceeds image token budget"); + require(patches.size()==size_t(n)*588,"patch count/shape mismatch"); + require(!detail::hip_bias_workspace(impl_->backend) || n>=16, + "HIP source-order vision normalization requires at least 16 patches"); + for(float value:patches) require(std::isfinite(value),"non-finite image patch"); + std::vector x; + { + Graph g; + auto input=ggml_new_tensor_2d(g.c,GGML_TYPE_F32,588,n); + auto y=impl_->linear(g.c,rounded(g.c,input),"vision.patch_embed.proj"); + g.stage("patch_embed",y,bool(observer)); + x=impl_->execute(g,input,patches,y,observer); + } + std::vector cos_values,sin_values; + // Patch output is already on the host. Release its reusable device arena + // before the independently bounded rotary temporary allocation. + if(detail::hip_bias_workspace(impl_->backend)) release_scratch(); + detail::rotary_tables(grid,cos_values,sin_values,impl_->backend); + for(int i=0;i<32;++i) { + Graph g; + const auto p="vision.blocks."+std::to_string(i); + auto input=ggml_new_tensor_2d(g.c,GGML_TYPE_F32,1024,n); + auto cosine=ggml_new_tensor_3d(g.c,GGML_TYPE_F32,32,1,n); + auto sine=ggml_new_tensor_3d(g.c,GGML_TYPE_F32,32,1,n); + ggml_set_input(cosine); ggml_set_input(sine); + auto normalized=impl_->norm(g.c,input,p+".norm1"); + auto qkv=impl_->linear(g.c,normalized,p+".attn.wqkv"); + auto slice=[&](int offset) { + return ggml_cont(g.c,ggml_view_3d(g.c,qkv,64,16,n,64*sizeof(float),3072*sizeof(float),offset*1024*sizeof(float))); + }; + auto q=detail::rotate(g.c,slice(0),cosine,sine); + auto k=detail::rotate(g.c,slice(1),cosine,sine); + auto attention=detail::attention(g.c,q,k,slice(2),impl_->backend); + auto projected=impl_->linear(g.c,attention,p+".attn.wo"); + auto residual=rounded(g.c,ggml_add(g.c,input,projected)); + auto mlp=impl_->linear(g.c,impl_->norm(g.c,residual,p+".norm2"),p+".mlp.w1",false); + auto gate=ggml_cont(g.c,ggml_view_2d(g.c,mlp,2816,n,5632*sizeof(float),0)); + auto up=ggml_view_2d(g.c,mlp,2816,n,5632*sizeof(float),2816*sizeof(float)); + auto product=rounded(g.c,ggml_mul(g.c,rounded(g.c,ggml_silu(g.c,gate)),up)); + auto y=rounded(g.c,ggml_add(g.c,residual,impl_->linear(g.c,product,p+".mlp.w2",false))); + if(i==0) { + g.stage("block0.norm1",normalized,bool(observer)); g.stage("block0.qkv",qkv,bool(observer)); + g.stage("block0.q",q,bool(observer)); g.stage("block0.k",k,bool(observer)); + g.stage("block0.attention",attention,bool(observer)); + } + g.stage("block"+std::to_string(i),y,bool(observer)); + x=impl_->execute(g,input,x,y,observer,cosine,sine,cos_values,sin_values); + } + { + Graph g; + auto input=ggml_new_tensor_2d(g.c,GGML_TYPE_F32,1024,n); + auto y=impl_->norm(g.c,input,"vision.norm"); + g.stage("features",y,bool(observer)); + x=impl_->execute(g,input,x,y,observer); + } + VisionOutput result; + if(retain) result.features=x; + { + Graph g; + auto input=ggml_new_tensor_2d(g.c,GGML_TYPE_F32,1024,n); + auto unfolded=detail::unfold(g.c,input,grid,1024); + auto first=impl_->linear(g.c,unfolded,"aligner.w1"); + auto activated=rounded(g.c,ggml_gelu_erf(g.c,first)); + auto y=impl_->linear(g.c,activated,"aligner.w2"); + g.stage("unfold",unfolded,bool(observer)); g.stage("aligner.w1",first,bool(observer)); + g.stage("aligner.gelu",activated,bool(observer)); g.stage("embeddings",y,bool(observer)); + result.embeddings=impl_->execute(g,input,x,y,observer); + } + for(float value:result.embeddings) require(std::isfinite(value),"non-finite vision output"); + result.rows=int(rows); result.columns=4096; output=std::move(result); + return true; + } catch(const std::exception & e) { error=e.what(); return false; } +} +bool VisionRuntime::sentinel(Sentinel identity,std::vector & output,std::string & error) const { + error.clear(); output.clear(); + if(!impl_) { error="vision runtime is not loaded"; return false; } + const char * name=nullptr; + switch(identity) { + case Sentinel::Start: name="image_start"; break; case Sentinel::Pad: name="image_pad"; break; + case Sentinel::Newline: name="image_newline"; break; case Sentinel::End: name="image_end"; break; + default: error="invalid sentinel identity"; return false; + } + std::vector raw(4096); + ggml_backend_tensor_get(impl_->weight(name),raw.data(),0,raw.size()*sizeof(ggml_bf16_t)); + output.resize(4096); + ggml_bf16_to_fp32_row(raw.data(),output.data(),int64_t(raw.size())); + return true; +} +void VisionRuntime::release_scratch() { if(impl_ && impl_->allocator) { ggml_gallocr_free(impl_->allocator); impl_->allocator=nullptr; } } +const VisionConfig * VisionRuntime::config() const { return impl_ ? &impl_->config : nullptr; } +size_t VisionRuntime::weight_bytes() const { return impl_ ? ggml_backend_buffer_get_size(impl_->buffer) : 0; } +size_t VisionRuntime::scratch_bytes() const { + if(!impl_) return 0; + // This conservative reservation remains after release_scratch(): the HIP + // context retains its workspace until backend destruction. + return detail::hip_bias_workspace(impl_->backend) + + (impl_->allocator ? ggml_gallocr_get_buffer_size(impl_->allocator,0) : 0); +} +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_vision.h b/server/src/deepseek4/deepseek4_vision.h new file mode 100644 index 000000000..746ed22a7 --- /dev/null +++ b/server/src/deepseek4/deepseek4_vision.h @@ -0,0 +1,80 @@ +#pragma once + +#include "ggml-backend.h" +#include +#include +#include +#include +#include + +namespace luce::vision { + +struct VisionConfig { + int layers = 32, dimension = 1024, heads = 16, intermediate = 2816; + int patch_size = 14, downsample = 3, language_dimension = 4096, vocabulary = 129280; + int max_image_tokens = 384; + float rope_theta = 10000.0f, rms_epsilon = 1e-6f; +}; +struct PatchGrid { int height, width; }; +enum class Sentinel { Start, Pad, Newline, End }; +struct VisionOutput { + int rows = 0, columns = 0; + std::vector embeddings; + // Optional raster [height * width, 1024] qualification output. + std::vector features; +}; +// Callback receives row-major values with the GGML shape (fastest axis first). +// It is synchronous: retaining diagnostics is the caller's responsibility. +using StageObserver = std::function &, + const std::vector &)>; + +// Owns weights and reusable graph allocator; borrows the backend, which must +// outlive this object. Sequential use only. No image decoding or token layout. +class VisionRuntime { +public: + VisionRuntime(); + ~VisionRuntime(); + VisionRuntime(const VisionRuntime &) = delete; + VisionRuntime & operator=(const VisionRuntime &) = delete; + bool load(const std::string & path, ggml_backend_t backend, + int language_dimension, int vocabulary, std::string & error); + bool encode(const std::vector & channel_major_patches, PatchGrid grid, + VisionOutput & output, std::string & error, bool retain_features = false, + const StageObserver & observer = {}); + bool sentinel(Sentinel identity, std::vector & output, std::string & error) const; + // Releases graph arena; backend-owned HIP Lt workspace remains retained. + void release_scratch(); + const VisionConfig * config() const; + size_t weight_bytes() const; + // Conservative graph arena + external HIP workspace reservation. + size_t scratch_bytes() const; +private: + struct Impl; + std::unique_ptr impl_; +}; + +// The same geometry primitives used by the runtime and standalone unit tests. +namespace detail { +// HIP registry capability; zero on CPU/NVIDIA or an unavailable backend. +size_t hip_bias_workspace(ggml_backend_t); +size_t hip_bias_launches(ggml_backend_t); +bool hip_norm_capable(ggml_backend_t); +size_t hip_norm_launches(ggml_backend_t); +bool hip_rotary_capable(ggml_backend_t); +size_t hip_rotary_launches(ggml_backend_t); +bool hip_softmax_capable(ggml_backend_t); +size_t hip_softmax_launches(ggml_backend_t); +bool hip_av_capable(ggml_backend_t); +size_t hip_av_launches(ggml_backend_t); +// F32 values exactly representing BF16 inputs; HIP source order needs at least 16 rows. +ggml_tensor * rms_norm(ggml_context *, ggml_tensor * input, float epsilon, ggml_backend_t); +ggml_tensor * linear(ggml_context *, ggml_tensor * weight, ggml_tensor * input, ggml_tensor * bias, + bool preserve_biased_product, ggml_backend_t backend = nullptr); +void rotary_tables(PatchGrid grid, std::vector & cosine, std::vector & sine, + ggml_backend_t backend = nullptr); +ggml_tensor * rotate(ggml_context *, ggml_tensor *, ggml_tensor * cosine, ggml_tensor * sine); +ggml_tensor * attention(ggml_context *, ggml_tensor * q, ggml_tensor * k, ggml_tensor * v, + ggml_backend_t backend = nullptr); +ggml_tensor * unfold(ggml_context *, ggml_tensor *, PatchGrid, int channels); +} +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_vision_preprocess.cpp b/server/src/deepseek4/deepseek4_vision_preprocess.cpp new file mode 100644 index 000000000..e701fb024 --- /dev/null +++ b/server/src/deepseek4/deepseek4_vision_preprocess.cpp @@ -0,0 +1,582 @@ +#include "deepseek4_vision_preprocess.h" +#include "../common/vision/image_resize.h" + +#include +#include +#include +#include +#include +#include + +namespace luce::vision { +namespace { + + +PreprocessStatus ok() { + return {}; +} + +PreprocessStatus fail(PreprocessError code, std::string message) { + return {code, std::move(message)}; +} + +bool checked_mul(std::uint64_t a, std::uint64_t b, std::uint64_t & out) { + if (a != 0 && b > std::numeric_limits::max() / a) { + return false; + } + out = a * b; + return true; +} + +bool checked_add(std::uint64_t a, std::uint64_t b, std::uint64_t & out) { + if (b > std::numeric_limits::max() - a) { + return false; + } + out = a + b; + return true; +} + +std::uint64_t round_ties_to_even(double value) { + const double base_double = std::floor(value); + const double fraction = value - base_double; + const auto base = static_cast(base_double); + if (fraction > 0.5 || (fraction == 0.5 && (base & 1U) != 0)) { + return base + 1; + } + return base; +} + +struct GridTokens { + std::uint32_t rows = 0; + std::uint32_t cols = 0; + std::uint64_t tokens = 0; +}; + +PreprocessStatus grid_tokens( + std::uint32_t height, + std::uint32_t width, + const PreprocessConfig & config, + GridTokens & result) { + if (height == 0 || width == 0 || height % config.patch_size != 0 || + width % config.patch_size != 0) { + return fail(PreprocessError::ResizePlanFailed, + "planned dimensions must be nonzero patch multiples"); + } + + const std::uint64_t vit_rows = height / config.patch_size; + const std::uint64_t vit_cols = width / config.patch_size; + const std::uint64_t rows = + (vit_rows + config.downsample_ratio - 1) / config.downsample_ratio; + const std::uint64_t cols = + (vit_cols + config.downsample_ratio - 1) / config.downsample_ratio; + if (rows == 0 || cols == 0 || rows > std::numeric_limits::max() || + cols > std::numeric_limits::max()) { + return fail(PreprocessError::ResizePlanFailed, "aligner grid is out of range"); + } + + std::uint64_t row_len = 0; + std::uint64_t tokens = 0; + if (!checked_add(cols, 1, row_len) || !checked_mul(rows, row_len, tokens) || + !checked_add(tokens, 2, tokens)) { + return fail(PreprocessError::ResizePlanFailed, "layout token count overflow"); + } + if ((rows & 1U) != 0 && !checked_add(tokens, row_len, tokens)) { + return fail(PreprocessError::ResizePlanFailed, "layout row padding overflow"); + } + const std::uint64_t trailing = (((rows + 1) / 2) * row_len % 2) * 2; + if (!checked_add(tokens, trailing, tokens)) { + return fail(PreprocessError::ResizePlanFailed, "layout tail padding overflow"); + } + + result.rows = static_cast(rows); + result.cols = static_cast(cols); + result.tokens = tokens; + return ok(); +} + +PreprocessStatus solve_resize_ratio( + std::uint32_t height, + std::uint32_t width, + std::uint32_t max_tokens, + const PreprocessConfig & config, + std::uint32_t & best_height, + std::uint32_t & best_width, + GridTokens & grid) { + if (max_tokens <= 2 || height == 0 || width == 0) { + return fail(PreprocessError::ResizePlanFailed, "resize budget is too small"); + } + + const double ratio = static_cast(height) / static_cast(width); + const double max_width_float = + std::sqrt((static_cast(max_tokens) - 2.0) / ratio + 0.25) - 0.5; + const double max_height_float = max_width_float * ratio; + std::uint64_t solved_width = 0; + std::uint64_t solved_height = 0; + const std::uint64_t stride = + static_cast(config.patch_size) * config.downsample_ratio; + + if (max_width_float < 1.0) { + const std::uint64_t max_width = 1; + std::uint64_t max_height = (max_tokens - 2) / (max_width + 1); + if ((max_height & 1U) != 0) { + --max_height; + } + solved_width = max_width * stride; + solved_height = max_height * stride; + } else if (max_height_float < 2.0) { + const std::uint64_t max_height = 2; + const std::uint64_t max_width = (max_tokens - 2) / max_height - 1; + if (max_width <= 1) { + return fail(PreprocessError::ResizePlanFailed, + "resize budget cannot represent a wide image"); + } + solved_width = max_width * stride; + solved_height = max_height * stride; + } else { + const auto max_width = static_cast(std::floor(max_width_float)); + auto max_height = static_cast(std::floor(max_height_float)); + if ((max_height & 1U) != 0) { + --max_height; + } + if (max_width == 0 || max_height == 0) { + return fail(PreprocessError::ResizePlanFailed, "resize solver produced an empty grid"); + } + const double beta = std::min( + static_cast(max_width * stride) / static_cast(width), + static_cast(max_height * stride) / static_cast(height)); + solved_width = static_cast( + std::floor(static_cast(width) * beta / config.patch_size)) * + config.patch_size; + solved_height = static_cast( + std::floor(static_cast(height) * beta / config.patch_size)) * + config.patch_size; + } + + if (solved_width == 0 || solved_height == 0 || + solved_width > std::numeric_limits::max() || + solved_height > std::numeric_limits::max()) { + return fail(PreprocessError::ResizePlanFailed, + "resize solver produced out-of-range dimensions"); + } + best_width = static_cast(solved_width); + best_height = static_cast(solved_height); + return grid_tokens(best_height, best_width, config, grid); +} + +PreprocessStatus pillow_resize( + const std::vector & input, + int input_width, + int input_height, + int output_width, + int output_height, + std::vector & output) { + std::string message; + if (!resize_rgb_bicubic(input, input_width, input_height, output_width, output_height, output, message)) { + return fail(PreprocessError::ResizePlanFailed, message); + } + return ok(); +} + +PreprocessStatus resize_and_pad( + const std::vector & input, + int input_width, + int input_height, + int output_width, + int output_height, + bool direct_resize, + std::vector & output) { + if (direct_resize) { + return pillow_resize( + input, input_width, input_height, output_width, output_height, output); + } + + int contained_width = output_width; + int contained_height = output_height; + const double input_ratio = static_cast(input_width) / input_height; + const double destination_ratio = static_cast(output_width) / output_height; + if (input_ratio != destination_ratio) { + if (input_ratio > destination_ratio) { + contained_height = static_cast(round_ties_to_even( + static_cast(input_height) / input_width * output_width)); + } else { + contained_width = static_cast(round_ties_to_even( + static_cast(input_width) / input_height * output_height)); + } + } + if (contained_width <= 0 || contained_height <= 0) { + return fail(PreprocessError::ResizePlanFailed, + "ImageOps.contain produced an empty dimension"); + } + + std::vector contained; + if (const auto status = pillow_resize( + input, input_width, input_height, contained_width, contained_height, contained); + !status) { + return status; + } + if (contained_width == output_width && contained_height == output_height) { + output = std::move(contained); + return ok(); + } + + output.assign(static_cast(output_width) * output_height * 3, 127); + const int offset_x = contained_width == output_width + ? 0 + : static_cast(round_ties_to_even((output_width - contained_width) * 0.5)); + const int offset_y = contained_width != output_width + ? 0 + : static_cast(round_ties_to_even((output_height - contained_height) * 0.5)); + for (int y = 0; y < contained_height; ++y) { + const auto * source = contained.data() + static_cast(y) * contained_width * 3; + auto * destination = output.data() + + (static_cast(y + offset_y) * output_width + offset_x) * 3; + std::memcpy(destination, source, static_cast(contained_width) * 3); + } + return ok(); +} + +std::uint16_t float_to_bf16(float value) { + std::uint32_t bits = 0; + static_assert(sizeof(bits) == sizeof(value)); + std::memcpy(&bits, &value, sizeof(bits)); + const std::uint32_t rounding_bias = 0x7FFFU + ((bits >> 16U) & 1U); + return static_cast((bits + rounding_bias) >> 16U); +} + +std::uint16_t normalize_pixel(std::uint8_t pixel) { + // Keep the source operation sequence and its F32 rounding points. + volatile float normalized = static_cast(pixel); + normalized = normalized / 255.0F; + normalized = normalized - 0.5F; + normalized = normalized / 0.5F; + return float_to_bf16(normalized); +} + +PreprocessStatus make_patches( + const std::vector & rgb, + const ResizePlan & plan, + const PreprocessConfig & config, + std::vector & patches) { + std::uint64_t pixel_count = 0; + std::uint64_t value_count = 0; + if (!checked_mul(plan.resized_width, plan.resized_height, pixel_count) || + !checked_mul(pixel_count, 3, value_count) || + value_count > std::numeric_limits::max()) { + return fail(PreprocessError::OutputTooLarge, "patch tensor size overflow"); + } + if (rgb.size() != value_count) { + return fail(PreprocessError::ResizePlanFailed, "resized RGB buffer has the wrong size"); + } + patches.resize(static_cast(value_count)); + + const std::uint32_t patch = config.patch_size; + std::size_t destination = 0; + for (std::uint32_t patch_y = 0; patch_y < plan.vit_rows; ++patch_y) { + for (std::uint32_t patch_x = 0; patch_x < plan.vit_cols; ++patch_x) { + for (std::uint32_t channel = 0; channel < 3; ++channel) { + for (std::uint32_t y = 0; y < patch; ++y) { + for (std::uint32_t x = 0; x < patch; ++x) { + const std::uint32_t source_y = patch_y * patch + y; + const std::uint32_t source_x = patch_x * patch + x; + const std::size_t source = + (static_cast(source_y) * plan.resized_width + source_x) * + 3 + + channel; + patches[destination++] = normalize_pixel(rgb[source]); + } + } + } + } + } + return ok(); +} + +} // namespace + +PreprocessStatus validate_config(const PreprocessConfig & config) { + const PreprocessConfig expected; + if (config.patch_size != expected.patch_size || + config.downsample_ratio != expected.downsample_ratio || + config.max_tokens != expected.max_tokens || + config.min_pixels != expected.min_pixels || + config.max_aspect_ratio != expected.max_aspect_ratio || + config.compress_pad_to != expected.compress_pad_to || + config.vocab_size != expected.vocab_size || + config.normalization_mean != expected.normalization_mean || + config.normalization_std != expected.normalization_std) { + return fail(PreprocessError::InvalidConfig, + "preprocessing config does not match the fixed DeepSeek-V4 vision recipe"); + } + return ok(); +} + +PreprocessStatus validate_decoded_dimensions( + std::uint32_t width, + std::uint32_t height, + const PreprocessLimits & limits) { + if (width == 0 || height == 0) { + return fail(PreprocessError::InvalidDimensions, "decoded RGB dimensions must be positive"); + } + if (limits.max_decoded_pixels == 0 || limits.max_dimension == 0 || + limits.max_output_pixels == 0) { + return fail(PreprocessError::InvalidConfig, "preprocessing limits must be positive"); + } + if (width > limits.max_dimension || height > limits.max_dimension) { + return fail(PreprocessError::InputTooLarge, "decoded RGB dimension exceeds the limit"); + } + std::uint64_t pixels = 0; + if (!checked_mul(width, height, pixels) || pixels > limits.max_decoded_pixels) { + return fail(PreprocessError::InputTooLarge, "decoded RGB pixel count exceeds the limit"); + } + return ok(); +} + +PreprocessStatus plan_image( + std::uint32_t width, + std::uint32_t height, + ResizePlan & plan, + const PreprocessConfig & config, + const PreprocessLimits & limits) { + if (const auto status = validate_config(config); !status) { + return status; + } + if (const auto status = validate_decoded_dimensions(width, height, limits); !status) { + return status; + } + + const bool direct_resize = + static_cast(width) >= + static_cast(height) * config.max_aspect_ratio; + std::uint64_t planned_width = width; + std::uint64_t planned_height = height; + const std::uint64_t max_width = + static_cast(height) * config.max_aspect_ratio; + if (planned_width > max_width) { + planned_width = max_width; + } + + std::uint64_t planned_pixels = 0; + if (!checked_mul(planned_width, planned_height, planned_pixels)) { + return fail(PreprocessError::ResizePlanFailed, "planned pixel count overflow"); + } + if (planned_pixels > 0 && planned_pixels < config.min_pixels) { + const double ratio = std::sqrt( + static_cast(config.min_pixels) / static_cast(planned_pixels)); + planned_width = static_cast(planned_width * ratio); + planned_height = static_cast(planned_height * ratio); + } + if (planned_width == 0 || planned_height == 0 || + planned_width > std::numeric_limits::max() || + planned_height > std::numeric_limits::max()) { + return fail(PreprocessError::ResizePlanFailed, + "minimum-pixel scaling produced invalid dimensions"); + } + + const std::uint64_t aligned_width = + ((planned_width + config.patch_size - 1) / config.patch_size) * config.patch_size; + const std::uint64_t aligned_height = + ((planned_height + config.patch_size - 1) / config.patch_size) * config.patch_size; + if (aligned_width > std::numeric_limits::max() || + aligned_height > std::numeric_limits::max()) { + return fail(PreprocessError::ResizePlanFailed, "aligned dimensions are out of range"); + } + std::uint32_t best_width = static_cast(aligned_width); + std::uint32_t best_height = static_cast(aligned_height); + GridTokens grid; + if (const auto status = grid_tokens(best_height, best_width, config, grid); !status) { + return status; + } + + const std::uint32_t reserved = config.compress_pad_to - 1; + if (config.max_tokens <= reserved + 2) { + return fail(PreprocessError::InvalidConfig, "token budget cannot hold image sentinels"); + } + const std::uint32_t image_budget = config.max_tokens - reserved; + std::uint32_t solver_budget = image_budget; + std::uint32_t attempts = 0; + while (grid.tokens > image_budget) { + if (solver_budget <= 2 || ++attempts > config.max_tokens) { + return fail(PreprocessError::ResizePlanFailed, + "could not solve image dimensions within the token budget"); + } + if (const auto status = solve_resize_ratio( + static_cast(planned_height), + static_cast(planned_width), + solver_budget, + config, + best_height, + best_width, + grid); + !status) { + return status; + } + --solver_budget; + } + + std::uint64_t output_pixels = 0; + if (!checked_mul(best_width, best_height, output_pixels) || + output_pixels > limits.max_output_pixels) { + return fail(PreprocessError::OutputTooLarge, + "planned resized image exceeds the output pixel limit"); + } + plan.resized_width = best_width; + plan.resized_height = best_height; + plan.vit_rows = best_height / config.patch_size; + plan.vit_cols = best_width / config.patch_size; + plan.aligner_rows = grid.rows; + plan.aligner_cols = grid.cols; + plan.direct_resize = direct_resize; + return ok(); +} + +PreprocessStatus build_image_layout( + std::uint32_t aligner_rows, + std::uint32_t aligner_cols, + std::uint64_t start_position, + ImageLayout & layout, + const PreprocessConfig & config) { + if (const auto status = validate_config(config); !status) { + return status; + } + if (aligner_rows == 0 || aligner_cols == 0) { + return fail(PreprocessError::InvalidDimensions, "aligner grid dimensions must be positive"); + } + + const std::uint64_t leading_pad = + config.compress_pad_to - 1 - start_position % config.compress_pad_to; + const std::uint64_t padded_rows = + static_cast(aligner_rows) + (aligner_rows % 2); + const std::uint64_t row_length = static_cast(aligner_cols) + 1; + std::uint64_t body_size = 0; + if (!checked_mul(padded_rows, row_length, body_size)) { + return fail(PreprocessError::TokenBudgetExceeded, "layout body size overflow"); + } + const std::uint64_t trailing_pad = (padded_rows / 2 * row_length % 2) * 2; + std::uint64_t total = 0; + if (!checked_add(leading_pad, 1, total) || !checked_add(total, body_size, total) || + !checked_add(total, trailing_pad, total) || !checked_add(total, 1, total)) { + return fail(PreprocessError::TokenBudgetExceeded, "layout token count overflow"); + } + if (total > config.max_tokens) { + std::ostringstream message; + message << "image layout needs " << total << " tokens, limit is " << config.max_tokens; + return fail(PreprocessError::TokenBudgetExceeded, message.str()); + } + std::uint64_t block_end = 0; + std::uint64_t visible_begin = 0; + if (!checked_add(start_position, total, block_end) || + !checked_add(start_position, leading_pad, visible_begin)) { + return fail(PreprocessError::PositionOverflow, "absolute image span overflow"); + } + + layout.types.clear(); + layout.permutation.clear(); + layout.types.reserve(static_cast(total)); + layout.permutation.reserve(static_cast(aligner_rows) * aligner_cols); + layout.types.insert(layout.types.end(), static_cast(leading_pad), + ImageTokenType::Pad); + layout.types.push_back(ImageTokenType::Start); + + for (std::uint64_t pair = 0; pair < padded_rows / 2; ++pair) { + for (std::uint64_t column = 0; column < row_length; ++column) { + for (std::uint64_t within_pair = 0; within_pair < 2; ++within_pair) { + const std::uint64_t row = pair * 2 + within_pair; + if (row < aligner_rows && column < aligner_cols) { + layout.types.push_back(ImageTokenType::Image); + layout.permutation.push_back(static_cast( + row * aligner_cols + column)); + } else if (row < aligner_rows && column == aligner_cols) { + layout.types.push_back(ImageTokenType::Newline); + } else { + layout.types.push_back(ImageTokenType::Pad); + } + } + } + } + layout.types.insert(layout.types.end(), static_cast(trailing_pad), + ImageTokenType::Pad); + layout.types.push_back(ImageTokenType::End); + layout.span = {start_position, visible_begin, block_end, block_end}; + return ok(); +} + +PreprocessResult preprocess_rgb( + const DecodedRgbView & input, + std::uint64_t start_position, + const PreprocessConfig & config, + const PreprocessLimits & limits) { + PreprocessResult result; + if (const auto status = validate_config(config); !status) { + result.status = status; + return result; + } + if (const auto status = validate_decoded_dimensions(input.width, input.height, limits); + !status) { + result.status = status; + return result; + } + std::uint64_t pixels = 0; + std::uint64_t expected_bytes = 0; + if (!checked_mul(input.width, input.height, pixels) || + !checked_mul(pixels, 3, expected_bytes) || input.data == nullptr || + expected_bytes != input.size) { + result.status = fail(PreprocessError::InputSizeMismatch, + "decoded RGB byte count must equal width * height * 3"); + return result; + } + if (const auto status = plan_image( + input.width, input.height, result.image.plan, config, limits); + !status) { + result.status = status; + return result; + } + if (const auto status = build_image_layout( + result.image.plan.aligner_rows, + result.image.plan.aligner_cols, + start_position, + result.image.layout, + config); + !status) { + result.status = status; + return result; + } + + std::vector owned_input(input.data, input.data + input.size); + if (const auto status = resize_and_pad( + owned_input, + static_cast(input.width), + static_cast(input.height), + static_cast(result.image.plan.resized_width), + static_cast(result.image.plan.resized_height), + result.image.plan.direct_resize, + result.image.resized_rgb); + !status) { + result.status = status; + return result; + } + if (const auto status = make_patches( + result.image.resized_rgb, result.image.plan, config, result.image.patches_bf16); + !status) { + result.status = status; + return result; + } + result.status = ok(); + return result; +} + +const char * preprocess_error_name(PreprocessError error) { + switch (error) { + case PreprocessError::None: return "none"; + case PreprocessError::InvalidConfig: return "invalid_config"; + case PreprocessError::InvalidDimensions: return "invalid_dimensions"; + case PreprocessError::InputSizeMismatch: return "input_size_mismatch"; + case PreprocessError::InputTooLarge: return "input_too_large"; + case PreprocessError::ResizePlanFailed: return "resize_plan_failed"; + case PreprocessError::OutputTooLarge: return "output_too_large"; + case PreprocessError::TokenBudgetExceeded: return "token_budget_exceeded"; + case PreprocessError::PositionOverflow: return "position_overflow"; + } + return "unknown"; +} + +} // namespace luce::vision diff --git a/server/src/deepseek4/deepseek4_vision_preprocess.h b/server/src/deepseek4/deepseek4_vision_preprocess.h new file mode 100644 index 000000000..d9299b495 --- /dev/null +++ b/server/src/deepseek4/deepseek4_vision_preprocess.h @@ -0,0 +1,122 @@ +#pragma once + +#include "../common/vision/image_decode.h" +#include "../common/vision/image_spans.h" + +#include +#include +#include +#include + +namespace luce::vision { + +struct PreprocessConfig { + std::uint32_t patch_size = 14; + std::uint32_t downsample_ratio = 3; + std::uint32_t max_tokens = 384; + std::uint64_t min_pixels = 147456; + std::uint32_t max_aspect_ratio = 8; + std::uint32_t compress_pad_to = 4; + std::uint32_t vocab_size = 129280; + float normalization_mean = 0.5F; + float normalization_std = 0.5F; +}; + +struct PreprocessLimits { + // The RGB entrypoint receives caller-owned memory, but validates these limits + // before allocating resized or patch buffers. A decoder can use the same + // dimension check before allocating its decoded image. + std::uint64_t max_decoded_pixels = 64ULL * 1024ULL * 1024ULL; + std::uint32_t max_dimension = 65'535; + std::uint64_t max_output_pixels = 16ULL * 1024ULL * 1024ULL; +}; + +enum class ImageTokenType : std::int64_t { + Start = 0, + Pad = 1, + Image = 2, + Newline = 3, + End = 4, +}; + +struct ResizePlan { + std::uint32_t resized_width = 0; + std::uint32_t resized_height = 0; + std::uint32_t vit_rows = 0; + std::uint32_t vit_cols = 0; + std::uint32_t aligner_rows = 0; + std::uint32_t aligner_cols = 0; + bool direct_resize = false; +}; + +struct ImageLayout { + std::vector types; + std::vector permutation; + TokenSpan span; +}; + +struct PreparedImage { + ResizePlan plan; + std::vector resized_rgb; + // Raw IEEE bfloat16 words in native integer representation. The tensor + // shape is [vit_rows * vit_cols, 3, patch_size, patch_size]. + std::vector patches_bf16; + ImageLayout layout; +}; + +enum class PreprocessError { + None = 0, + InvalidConfig, + InvalidDimensions, + InputSizeMismatch, + InputTooLarge, + ResizePlanFailed, + OutputTooLarge, + TokenBudgetExceeded, + PositionOverflow, +}; + +struct PreprocessStatus { + PreprocessError code = PreprocessError::None; + std::string message; + + explicit operator bool() const { return code == PreprocessError::None; } +}; + +struct PreprocessResult { + PreprocessStatus status; + PreparedImage image; + + explicit operator bool() const { return static_cast(status); } +}; + +PreprocessStatus validate_config(const PreprocessConfig & config); + +PreprocessStatus validate_decoded_dimensions( + std::uint32_t width, + std::uint32_t height, + const PreprocessLimits & limits = {}); + +PreprocessStatus plan_image( + std::uint32_t width, + std::uint32_t height, + ResizePlan & plan, + const PreprocessConfig & config = {}, + const PreprocessLimits & limits = {}); + +PreprocessStatus build_image_layout( + std::uint32_t aligner_rows, + std::uint32_t aligner_cols, + std::uint64_t start_position, + ImageLayout & layout, + const PreprocessConfig & config = {}); + +PreprocessResult preprocess_rgb( + const DecodedRgbView & input, + std::uint64_t start_position, + const PreprocessConfig & config = {}, + const PreprocessLimits & limits = {}); + +const char * preprocess_error_name(PreprocessError error); + +} // namespace luce::vision diff --git a/server/src/internal.h b/server/src/internal.h index 24e8792d3..3d8387008 100644 --- a/server/src/internal.h +++ b/server/src/internal.h @@ -243,6 +243,9 @@ struct TargetWeights { // comparands with `>= 0` so the sentinel never matches a real token. int32_t eos_id = -1; int32_t eos_chat_id = -1; + // Token the chat template repeats once per image token; -1 when the + // vocabulary has none (a model without image input). + int32_t image_pad_id = -1; // DFlash noise mask token ID (from target tokenizer, used by draft model). // Default: Qwen tokenizer's mask token. Overridden by GGUF metadata if available. @@ -268,6 +271,7 @@ struct TargetLoadPlan { bool skip_expert_tensors = false; // skip ffn_*_exps from GPU (for hybrid MoE split load) bool metadata_only = false; // parse tensor descriptors/scales without GPU allocation bool expert_metadata_only = false; // keep only routed expert tensor metadata; upload nothing + bool load_ds4_image_bias = false; }; // Load a Q4_K_M target model from a GGUF file on disk. diff --git a/server/src/qwen35/gguf_target_loader.cpp b/server/src/qwen35/gguf_target_loader.cpp index 06ad67c15..c3565c7aa 100644 --- a/server/src/qwen35/gguf_target_loader.cpp +++ b/server/src/qwen35/gguf_target_loader.cpp @@ -44,6 +44,7 @@ // tensor's bytes from the mmap'd file. #include "internal.h" +#include "qwen35_image_prompt.h" #include "common/derived_scalars.h" #include "common/gguf_inspect.h" #include "common/layer_split_utils.h" @@ -115,6 +116,18 @@ int32_t get_i32_or(const gguf_context * g, const char * key, int32_t fallback) { return gguf_get_val_i32(g, id); } +// Id of the vocabulary entry spelled exactly `text`, or -1. Searched from the +// end, where the added special tokens live. +static int32_t find_token_id(const gguf_context * g, const char * text) { + const int64_t key = gguf_find_key(g, "tokenizer.ggml.tokens"); + if (key < 0 || gguf_get_kv_type(g, key) != GGUF_TYPE_ARRAY || + gguf_get_arr_type(g, key) != GGUF_TYPE_STRING) return -1; + for (size_t i = gguf_get_arr_n(g, key); i-- > 0;) { + if (std::strcmp(gguf_get_arr_str(g, key, i), text) == 0) return (int32_t) i; + } + return -1; +} + uint32_t get_u32_or(const gguf_context * g, const char * key, uint32_t fallback) { int64_t id = gguf_find_key(g, key); if (id < 0) return fallback; @@ -544,6 +557,7 @@ bool load_target_gguf_partial(const std::string & path, out.eos_chat_id = (raw_eos_chat == kEosKeyMissing) ? -1 : (int32_t)raw_eos_chat; std::printf("[loader] eos_id=%d eos_chat_id=%d\n", out.eos_id, out.eos_chat_id); } + out.image_pad_id = find_token_id(gctx, QWEN35_IMAGE_PAD_TOKEN); // Compute capture layer IDs: evenly spaced through the target layers. // step = (n_layer - 2) / (N - 1), ids[k] = 1 + k * step. diff --git a/server/src/qwen35/qwen35_backend.cpp b/server/src/qwen35/qwen35_backend.cpp index c48b7562d..2dbbaa4dd 100644 --- a/server/src/qwen35/qwen35_backend.cpp +++ b/server/src/qwen35/qwen35_backend.cpp @@ -1,4 +1,5 @@ #include "qwen35_backend.h" +#include "qwen35_image_request.h" #include "concurrency/qwen35_seq_engine.h" #include "common/chain_rollback_policy.h" #include "common/adaptive_spec_width.h" @@ -328,6 +329,7 @@ bool Qwen35Backend::init() { return false; } std::printf("[target] %s\n", luce_last_error()); + if (!load_vision()) return false; if (cfg_.paged_attention && (w_.n_embd_head_k != 256 || w_.n_embd_head_v != 256)) { std::fprintf(stderr, @@ -894,6 +896,7 @@ bool Qwen35Backend::park(ParkTarget target) { step_graph_destroy(proj_sg_); dflash2_selector_graph_invalidate(); free_target_weights(w_); + vision_.reset(); target_parked_ = true; std::printf("[park] target released\n"); std::fflush(stdout); } @@ -910,6 +913,11 @@ bool Qwen35Backend::unpark(ParkTarget target) { std::fprintf(stderr, "[unpark] target: %s\n", luce_last_error()); return false; } + if (!load_vision()) { + // Stay parked, so a retry starts from released weights. + free_target_weights(w_); + return false; + } kvflash_drafter_failed_ = false; // fresh VRAM: allow a retry target_parked_ = false; std::printf("[unpark] target restored\n"); std::fflush(stdout); @@ -1337,6 +1345,7 @@ void Qwen35Backend::shutdown() { free_prefix_snapshot(prefix_snapshots_[i]); } if (!target_parked_) free_target_weights(w_); + vision_.reset(); // its buffers belong to target_backend_, freed below if (!use_remote_draft && !draft_parked_) free_draft_weights(dw_); free_target_cache(cache_); if (split_gpus_ && draft_backend_) { @@ -1437,7 +1446,19 @@ GenerateResult Qwen35Backend::generate_impl(const GenerateRequest & req, // Prefill auto t_prefill_start = std::chrono::steady_clock::now(); - const int committed = do_prefill(req.prompt, out_io, req.snap_pos, req.snap_slot); + Qwen35ImageRows image_rows; + if (req.images) { + const auto * prompt = dynamic_cast(req.images.get()); + std::string image_error = "image binding does not match the prompt"; + if (!prompt || !prompt->matches(req.prompt) || + !encode_images(*prompt, image_rows, image_error)) { + result.fail(GenerateErrorCode::BackendSpecific, image_error); + return result; + } + } + const bool has_images = image_rows.prompt != nullptr; + const int committed = do_prefill(req.prompt, out_io, req.snap_pos, req.snap_slot, + /*kv_offset=*/0, has_images ? &image_rows : nullptr); if (committed < 0) { result.fail(GenerateErrorCode::PrefillFailed); return result; @@ -1487,7 +1508,9 @@ GenerateResult Qwen35Backend::generate_impl(const GenerateRequest & req, req.n_gen, ar_n_gen, committed, cfg_.device.max_ctx); } } - if (cfg_.paged_attention || req.force_ar_decode) { + // Speculative decoding takes rotary positions from KV positions, which + // an image prompt pulls apart, so image requests decode one by one. + if (cfg_.paged_attention || req.force_ar_decode || has_images) { decode_ok = do_ar_decode(committed, ar_n_gen, result.tokens, out_io, req.budget_hook, &result.budget_forced_close, @@ -1539,6 +1562,16 @@ GenerateResult Qwen35Backend::restore_and_generate_impl(int slot, out_io.emit(-1); return result; } + if (req.images) { + // A snapshot is keyed by tokens, and pad tokens do not identify an image. + result.fail(GenerateErrorCode::BackendSpecific, + "image requests cannot resume from a snapshot"); + out_io.emit(-1); + return result; + } + // An exact snapshot hit decodes without a prefill, so the offset a + // previous image request left behind must not survive into this one. + rope_delta_ = 0; if (slot < 0 || slot >= PREFIX_SLOTS || !prefix_snapshots_[slot].ctx) { result.fail(GenerateErrorCode::InvalidSnapshotSlot); out_io.emit(-1); @@ -1709,7 +1742,19 @@ GenerateResult Qwen35Backend::restore_and_generate_impl(int slot, int Qwen35Backend::do_prefill(const std::vector & tokens, const DaemonIO & io, int snap_pos, int snap_slot, - int kv_offset) { + int kv_offset, + const Qwen35ImageRows * images) { + if (images && kv_offset != 0) { + std::fprintf(stderr, "prefill: an image prompt must start at position 0\n"); + return -1; + } + if (images) { + // Snapshots are found again by their tokens, and pads say nothing + // about which image they stood for. + snap_pos = -1; + snap_slot = -1; + } + rope_delta_ = images ? images->prompt->positions.next - (int)tokens.size() : 0; // A finite --fa-window caps the full-attention layers to a sliding // window, so anything earlier than the window is invisible to them. That // is silent: the model still answers, it just cannot see the head of a @@ -1901,12 +1946,17 @@ int Qwen35Backend::do_prefill(const std::vector & tokens, if (!w_.embedder.embed(tokens.data() + start, n_tokens, embed_buf.data())) { return -1; } + if (images) images->overwrite(embed_buf.data(), start, n_tokens, hidden); ggml_backend_tensor_set(sg_.inp_embed, embed_buf.data(), 0, sizeof(float) * (size_t)hidden * n_tokens); // Positions (M-RoPE) std::vector pos_buf((size_t)4 * n_tokens, 0); - fill_qwen35_mrope_positions(pos_buf.data(), kv_pos, n_tokens); + if (images) { + images->prompt->positions.fill(pos_buf.data(), start, n_tokens); + } else { + fill_qwen35_mrope_positions(pos_buf.data(), kv_pos, n_tokens); + } ggml_backend_tensor_set(sg_.positions, pos_buf.data(), 0, sizeof(int32_t) * pos_buf.size()); @@ -2310,7 +2360,8 @@ bool Qwen35Backend::do_ar_decode(int committed, int n_gen, if (!w_.embedder.embed(&tok, 1, embed_buf)) return false; ggml_backend_tensor_set(sg_.inp_embed, embed_buf, 0, sizeof(float) * hidden); - int32_t pos4[4] = {committed, committed, committed, 0}; + const int32_t rope_pos = committed + rope_delta_; + int32_t pos4[4] = {rope_pos, rope_pos, rope_pos, 0}; ggml_backend_tensor_set(sg_.positions, pos4, 0, sizeof(int32_t) * 4); // kvflash: graph carries a slot-validity mask alongside the diff --git a/server/src/qwen35/qwen35_backend.h b/server/src/qwen35/qwen35_backend.h index 67048227f..423bb95d8 100644 --- a/server/src/qwen35/qwen35_backend.h +++ b/server/src/qwen35/qwen35_backend.h @@ -26,6 +26,7 @@ #include "common/concurrency/paged_kv_pool.h" #include "concurrency/qwen35_seq_engine.h" #include "internal.h" // TargetWeights, TargetCache, DraftWeights, PrefixSnapshot +#include "qwen35_vision.h" #include "qwen3/qwen3_drafter.h" // DrafterContext, load_drafter, free_drafter, drafter_score_and_compress #include "kvflash_pager.h" // bounded KV residency pool #include "kvflash_scorer.h" // chunk-relevance policy interface @@ -43,12 +44,15 @@ namespace luce::common { class Qwen35TensorParallelContext; +class Qwen35ImagePrompt; +struct Qwen35ImageRows; // ── Configuration passed at construction ──────────────────────────────── struct Qwen35Config { std::string target_path; std::optional draft_path; + std::string mmproj_path; // vision projector; empty = text only DevicePlacement device; // target GPU placement int draft_gpu = 0; RemoteDraftConfig remote_draft; @@ -153,6 +157,18 @@ class Qwen35Backend : public ModelBackend { bool supports_dflash_spec_decode() const override { return !cfg_.paged_attention; } DFlashTarget * dflash_target() override; + + // Image input (--mmproj). Requests with images prefill through the normal + // chunk loop with the image rows written over the pad embeddings, and + // decode one token at a time. + bool supports_images() const override { return image_input_; } + std::string image_placeholder() const override; + bool prepare_images(std::vector & tokens, + std::vector images, + uint64_t context_capacity, + uint64_t output_reserve, + ImagePromptHandle & payload, + std::string & error) const override; bool supports_remote_draft() const override { return true; } // ── Concurrent slot serving (paged AR decode over N sequences) ──── @@ -291,6 +307,19 @@ class Qwen35Backend : public ModelBackend { bool target_parked_ = false; bool draft_parked_ = false; + // Vision projector, loaded next to the target weights when --mmproj is set. + bool load_vision(); + bool encode_images(const Qwen35ImagePrompt & prompt, Qwen35ImageRows & rows, + std::string & error); + std::unique_ptr vision_; // released while parked + // Fixed after init(); read by prepare_images() on request threads. + bool image_input_ = false; + vision::Qwen35VisionConfig vision_config_; + // Rotary position minus KV position for the sequence being decoded. + // Negative after an image prompt (an image spans fewer positions than + // tokens); zero for text. + int rope_delta_ = 0; + // ── Pflash drafter (lazy-loaded) ───────────────────────────────── DrafterContext drafter_ctx_; bool drafter_loaded_ = false; @@ -341,10 +370,12 @@ class Qwen35Backend : public ModelBackend { // Prefill a prompt and return the number of tokens committed to KV. // kv_offset > 0 resumes from a restored snapshot: tokens are placed at // KV positions [kv_offset, kv_offset + tokens.size()) instead of [0, N). + // `images` carries the encoded rows of an image prompt (kv_offset 0 only). int do_prefill(const std::vector & tokens, const DaemonIO & io, int snap_pos = -1, int snap_slot = -1, - int kv_offset = 0); + int kv_offset = 0, + const Qwen35ImageRows * images = nullptr); // Speculative decode loop: draft → verify → accept until EOS/max. // When budget_hook is non-null and (n_gen - generated) drops to the diff --git a/server/src/qwen35/qwen35_backend_images.cpp b/server/src/qwen35/qwen35_backend_images.cpp new file mode 100644 index 000000000..9020ebaee --- /dev/null +++ b/server/src/qwen35/qwen35_backend_images.cpp @@ -0,0 +1,125 @@ +// Image input for Qwen35Backend: projector loading, request preparation, and +// encoding. The prefill and decode changes live next to the code they touch +// in qwen35_backend.cpp. +#include "qwen35_backend.h" + +#include "common/vision/image_decode.h" +#include "qwen35_image_request.h" + +#include +#include +#include +#include + +namespace luce::common { + +namespace { +constexpr size_t MAX_IMAGES_PER_REQUEST = 4; // the server's transport limit +} + +bool Qwen35Backend::load_vision() { + if (cfg_.mmproj_path.empty()) return true; + if (w_.image_pad_id < 0) { + std::fprintf(stderr, "[vision] this model's vocabulary has no <|image_pad|> token\n"); + return false; + } + auto tower = std::make_unique(); + std::string error; + if (!tower->load(cfg_.mmproj_path, target_backend_, w_.n_embd, error)) { + std::fprintf(stderr, "[vision] %s\n", error.c_str()); + return false; + } + // Request threads read these two without a lock, so they are written + // once. A reload after unpark must bring back the same projector. + if (!image_input_) { + vision_config_ = tower->config(); + image_input_ = true; + } else if (!tower->config().same_geometry(vision_config_)) { + std::fprintf(stderr, "[vision] the projector file changed while the model was parked\n"); + return false; + } + vision_ = std::move(tower); + std::printf("[vision] projector loaded: %d layers, %.0f MiB, up to %d tokens per image\n", + vision_config_.layers, vision_->weight_bytes() / (1024.0 * 1024.0), + vision_config_.max_image_tokens); + return true; +} + +std::string Qwen35Backend::image_placeholder() const { + return image_input_ ? QWEN35_IMAGE_PLACEHOLDER : ""; +} + +bool Qwen35Backend::prepare_images(std::vector & tokens, std::vector images, + uint64_t context_capacity, uint64_t output_reserve, + ImagePromptHandle & payload, std::string & error) const { + payload.reset(); + if (images.empty()) { + // A pad typed into a text prompt would be embedded as an ordinary + // token, with no image behind it. + if (image_input_ && std::find(tokens.begin(), tokens.end(), w_.image_pad_id) != tokens.end()) { + error = "image marker in a prompt without images"; + return false; + } + return true; + } + if (!image_input_) { error = "this model was started without --mmproj"; return false; } + if (images.size() > MAX_IMAGES_PER_REQUEST) { error = "too many images in request"; return false; } + try { + auto prompt = std::make_shared(); + prompt->owner = this; + for (const EncodedImage & image : images) { + auto decoded = vision::decode_image({image.bytes.data(), image.bytes.size()}); + if (!decoded) { error = decoded.status.message; return false; } + vision::Qwen35Pixels pixels; + if (!vision::qwen35_vision_preprocess(vision_config_, decoded.image, pixels, error)) return false; + Qwen35ImageSlot slot; + slot.columns = pixels.grid_columns; + slot.rows = pixels.grid_rows; + prompt->slots.push_back(slot); + prompt->pixels.push_back(std::move(pixels)); + } + const uint64_t limit = context_capacity > output_reserve ? context_capacity - output_reserve : 0; + if (!qwen35_expand_image_tokens(tokens, w_.image_pad_id, prompt->slots, limit, error)) return false; + prompt->expanded_tokens = tokens; + prompt->positions = qwen35_image_rope_positions((int) tokens.size(), prompt->slots); + payload = std::move(prompt); + return true; + } catch (const std::bad_alloc &) { + error = "image preparation allocation failed"; + return false; + } +} + +bool Qwen35Backend::encode_images(const Qwen35ImagePrompt & prompt, Qwen35ImageRows & rows, + std::string & error) { + if (prompt.owner != this || !vision_) { + error = "image binding does not belong to the loaded backend"; + return false; + } + rows.prompt = &prompt; + bool ok = true; + int tokens = 0; + const auto start = std::chrono::steady_clock::now(); + try { + rows.rows.resize(prompt.pixels.size()); + for (size_t i = 0; ok && i < prompt.pixels.size(); ++i) { + ok = vision_->encode(prompt.pixels[i], rows.rows[i], error); + tokens += prompt.pixels[i].tokens(); + } + } catch (const std::bad_alloc &) { + error = "image encoding allocation failed"; + ok = false; + } + ggml_backend_synchronize(target_backend_); + const double ms = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + if (ok) { + std::printf("[vision] encoded %zu image(s), %d tokens, in %.0f ms\n", prompt.pixels.size(), tokens, ms); + std::fflush(stdout); + } + // The attention scratch is large and only needed here; give it back + // before prefill sizes its own graphs. + vision_->release_scratch(); + return ok; +} + +} // namespace luce::common diff --git a/server/src/qwen35/qwen35_image_prompt.cpp b/server/src/qwen35/qwen35_image_prompt.cpp new file mode 100644 index 000000000..898a80c67 --- /dev/null +++ b/server/src/qwen35/qwen35_image_prompt.cpp @@ -0,0 +1,84 @@ +#include "qwen35_image_prompt.h" + +#include +#include + +namespace luce::common { + +bool qwen35_expand_image_tokens(std::vector & tokens, int32_t image_pad, + std::vector & slots, uint64_t max_tokens, + std::string & error) { + const size_t pads = (size_t) std::count(tokens.begin(), tokens.end(), image_pad); + if (pads != slots.size()) { + error = "prompt has " + std::to_string(pads) + " image markers for " + + std::to_string(slots.size()) + " images"; + return false; + } + uint64_t total = tokens.size() - pads; + for (const Qwen35ImageSlot & slot : slots) { + if (slot.tokens() <= 0) { error = "image has no tokens"; return false; } + total += (uint64_t) slot.tokens(); + } + if (total > max_tokens) { + error = "prompt with images needs " + std::to_string(total) + " tokens, the limit is " + + std::to_string(max_tokens); + return false; + } + std::vector expanded; + expanded.reserve((size_t) total); + size_t next_slot = 0; + for (int32_t token : tokens) { + if (token != image_pad) { expanded.push_back(token); continue; } + Qwen35ImageSlot & slot = slots[next_slot++]; + slot.begin = (int) expanded.size(); + expanded.insert(expanded.end(), (size_t) slot.tokens(), image_pad); + } + tokens.swap(expanded); + return true; +} + +Qwen35RopePositions qwen35_image_rope_positions(int prompt_tokens, const std::vector & slots) { + Qwen35RopePositions out; + out.temporal.resize((size_t) prompt_tokens); + out.height.resize((size_t) prompt_tokens); + out.width.resize((size_t) prompt_tokens); + int position = 0, token = 0; + const auto text_until = [&](int end) { + for (; token < end; ++token, ++position) { + out.temporal[token] = out.height[token] = out.width[token] = position; + } + }; + for (const Qwen35ImageSlot & slot : slots) { + text_until(slot.begin); + for (int row = 0; row < slot.rows; ++row) { + for (int column = 0; column < slot.columns; ++column, ++token) { + out.temporal[token] = position; + out.height[token] = position + row; + out.width[token] = position + column; + } + } + position += std::max(slot.rows, slot.columns); + } + text_until(prompt_tokens); + out.next = position; + return out; +} + +void qwen35_overwrite_image_rows(const Qwen35ImageSlot & slot, const float * rows, int hidden, + float * embeddings, int first, int count) { + const int begin = std::max(first, slot.begin); + const int end = std::min(first + count, slot.begin + slot.tokens()); + if (begin >= end) return; + std::memcpy(embeddings + (size_t) (begin - first) * hidden, + rows + (size_t) (begin - slot.begin) * hidden, + sizeof(float) * (size_t) (end - begin) * hidden); +} + +void Qwen35RopePositions::fill(int32_t * out, int first, int count) const { + std::copy_n(temporal.begin() + first, count, out); + std::copy_n(height.begin() + first, count, out + count); + std::copy_n(width.begin() + first, count, out + 2 * count); + std::fill_n(out + 3 * count, count, 0); +} + +} // namespace luce::common diff --git a/server/src/qwen35/qwen35_image_prompt.h b/server/src/qwen35/qwen35_image_prompt.h new file mode 100644 index 000000000..362ebdfb2 --- /dev/null +++ b/server/src/qwen35/qwen35_image_prompt.h @@ -0,0 +1,52 @@ +// Prompt-side half of Qwen3.5 / Qwen3.8 image input: where each image sits in +// the token stream and which rotary positions its tokens take. Pure functions +// of the token ids and the image grids; no GPU, no projector. +#pragma once + +#include +#include +#include + +namespace luce::common { + +// What the chat template carries for one image. The middle token is the pad +// token, repeated once per image token by qwen35_expand_image_tokens(). +inline constexpr const char * QWEN35_IMAGE_PAD_TOKEN = "<|image_pad|>"; +inline constexpr const char * QWEN35_IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"; + +struct Qwen35ImageSlot { + int begin = 0; // first image token in the expanded prompt + int columns = 0, rows = 0; // image tokens across and down + + int tokens() const { return columns * rows; } +}; + +// `tokens` holds one `image_pad` per image, in order. Each becomes +// slot.tokens() pads and slot.begin is filled in. Fails when the number of +// pads differs from the number of slots or the result exceeds `max_tokens`. +bool qwen35_expand_image_tokens(std::vector & tokens, int32_t image_pad, + std::vector & slots, uint64_t max_tokens, + std::string & error); + +// Rotary positions of every prompt token. Text advances all three axes +// together. An image starting at position p holds the temporal axis at p and +// lays its rows and columns out from p on the other two; the text after it +// resumes at p + max(rows, columns). So positions run behind token indices +// after an image, by `next - prompt tokens`. +struct Qwen35RopePositions { + std::vector temporal, height, width; + int next = 0; // position of the first generated token + + // [4 * count] in the axis-major layout the target graph reads. + void fill(int32_t * out, int first, int count) const; +}; + +Qwen35RopePositions qwen35_image_rope_positions(int prompt_tokens, const std::vector & slots); + +// Copies the part of one image's `rows` that falls inside prompt tokens +// [first, first + count) over `embeddings`, which holds those `count` rows. +// Prefill runs in chunks, so an image can straddle several calls. +void qwen35_overwrite_image_rows(const Qwen35ImageSlot & slot, const float * rows, int hidden, + float * embeddings, int first, int count); + +} // namespace luce::common diff --git a/server/src/qwen35/qwen35_image_request.h b/server/src/qwen35/qwen35_image_request.h new file mode 100644 index 000000000..65a404c0b --- /dev/null +++ b/server/src/qwen35/qwen35_image_request.h @@ -0,0 +1,39 @@ +// What an image request carries through the Qwen3.5 backend: the prompt +// binding built on the request thread, and the encoded rows built on the +// worker thread just before prefill. +#pragma once + +#include "common/image_prompt.h" +#include "qwen35_image_prompt.h" +#include "qwen35_vision.h" + +#include +#include + +namespace luce::common { + +class Qwen35ImagePrompt final : public ImagePromptPayload { +public: + bool matches(const std::vector & tokens) const override { return tokens == expanded_tokens; } + + const void * owner = nullptr; // the backend that prepared it + std::vector expanded_tokens; // prompt with one pad per image token + std::vector slots; // one per image, in prompt order + std::vector pixels; // same order as slots + Qwen35RopePositions positions; +}; + +struct Qwen35ImageRows { + const Qwen35ImagePrompt * prompt = nullptr; + std::vector> rows; // per image: tokens() x hidden floats + + // Writes the image rows that fall inside prompt tokens [first, first + + // count) over `embeddings`, which holds `count` rows of `hidden` floats. + void overwrite(float * embeddings, int first, int count, int hidden) const { + for (size_t i = 0; i < rows.size(); ++i) { + qwen35_overwrite_image_rows(prompt->slots[i], rows[i].data(), hidden, embeddings, first, count); + } + } +}; + +} // namespace luce::common diff --git a/server/src/qwen35/qwen35_target_graph.cpp b/server/src/qwen35/qwen35_target_graph.cpp index 8f07e26d6..57d4249e3 100644 --- a/server/src/qwen35/qwen35_target_graph.cpp +++ b/server/src/qwen35/qwen35_target_graph.cpp @@ -1204,12 +1204,15 @@ static ggml_tensor * build_full_attn_block( int sections[4]; for (int i = 0; i < 4; i++) sections[i] = rope_sections[i]; + // Interleaved M-RoPE, as the model is defined. For text the three axes + // carry the same position and the layout makes no difference; image + // tokens give each axis its own value, and then it does. Q = ggml_rope_multi(ctx, Q, positions, /*freq_factors=*/nullptr, - n_rot, sections, GGML_ROPE_TYPE_MROPE, + n_rot, sections, GGML_ROPE_TYPE_IMROPE, /*n_ctx_orig=*/0, w.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); Kcur = ggml_rope_multi(ctx, Kcur, positions, nullptr, - n_rot, sections, GGML_ROPE_TYPE_MROPE, + n_rot, sections, GGML_ROPE_TYPE_IMROPE, 0, w.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); diff --git a/server/src/qwen35/qwen35_vision.cpp b/server/src/qwen35/qwen35_vision.cpp new file mode 100644 index 000000000..205894395 --- /dev/null +++ b/server/src/qwen35/qwen35_vision.cpp @@ -0,0 +1,382 @@ +#include "qwen35_vision.h" + +#include "../common/vision/image_resize.h" +#include "../common/vision/mmproj_file.h" + +#include "ggml-alloc.h" +#include "ggml.h" + +#include +#include +#include + +namespace luce::vision { + +namespace { + +constexpr const char * PROJECTOR_TYPE = "qwen3vl_merger"; +// The model's own preprocessing refuses anything more elongated. +constexpr int MAX_ASPECT_RATIO = 200; +constexpr size_t GRAPH_NODES = 2048; + +struct Block { + ggml_tensor * ln1_w, * ln1_b, * qkv_w, * qkv_b, * out_w, * out_b; + ggml_tensor * ln2_w, * ln2_b, * up_w, * up_b, * down_w, * down_b; +}; + +ggml_tensor * layer_norm(ggml_context * ctx, ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, float eps) { + return ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, eps), w), b); +} + +ggml_tensor * linear(ggml_context * ctx, ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) { + return ggml_add(ctx, ggml_mul_mat(ctx, w, x), b); +} + +struct ContextDeleter { + void operator()(ggml_context * ctx) const { ggml_free(ctx); } +}; + +} // namespace + +bool qwen35_vision_target_size(const Qwen35VisionConfig & config, int width, int height, + int & out_width, int & out_height, std::string & error) { + const int factor = config.patch_size * config.merge; + if (factor <= 0 || width <= 0 || height <= 0) { + error = "image has no pixels"; + return false; + } + if (std::max(width, height) > (int64_t) MAX_ASPECT_RATIO * std::min(width, height)) { + error = "image aspect ratio exceeds 200:1"; + return false; + } + const double area = double(width) * height; + const double min_pixels = double(config.min_image_tokens) * factor * factor; + const double max_pixels = double(config.max_image_tokens) * factor * factor; + // nearbyint rounds halves to even, like the reference implementation. + const auto nearest = [factor](double v) { return std::max(factor, int(std::nearbyint(v / factor)) * factor); }; + int w = nearest(width), h = nearest(height); + if (double(w) * h > max_pixels) { + const double shrink = std::sqrt(area / max_pixels); + w = std::max(factor, int(std::floor(width / shrink / factor)) * factor); + h = std::max(factor, int(std::floor(height / shrink / factor)) * factor); + } else if (double(w) * h < min_pixels) { + const double grow = std::sqrt(min_pixels / area); + w = int(std::ceil(width * grow / factor)) * factor; + h = int(std::ceil(height * grow / factor)) * factor; + } + out_width = w; + out_height = h; + return true; +} + +void qwen35_vision_position_rows(const std::vector & table, int side, int dimension, + int columns, int rows, int merge, std::vector & out) { + out.resize(size_t(columns) * rows * dimension); + // Aligned corners: point i of n lands on i * (side - 1) / (n - 1). + const auto source = [side](int i, int n) { return n > 1 ? double(i) * (side - 1) / (n - 1) : 0.0; }; + const auto row_of = [&](int y, int x) { return table.data() + (size_t(y) * side + x) * dimension; }; + float * dst = out.data(); + for (int by = 0; by < rows; by += merge) { + for (int bx = 0; bx < columns; bx += merge) { + for (int dy = 0; dy < merge; ++dy) { + for (int dx = 0; dx < merge; ++dx, dst += dimension) { + const double fy = source(by + dy, rows), fx = source(bx + dx, columns); + const int y0 = int(fy), x0 = int(fx); + const int y1 = std::min(y0 + 1, side - 1), x1 = std::min(x0 + 1, side - 1); + const float wy = float(fy - y0), wx = float(fx - x0); + const float * a = row_of(y0, x0), * b = row_of(y0, x1); + const float * c = row_of(y1, x0), * d = row_of(y1, x1); + for (int i = 0; i < dimension; ++i) { + dst[i] = (1.0f - wy) * ((1.0f - wx) * a[i] + wx * b[i]) + + wy * ((1.0f - wx) * c[i] + wx * d[i]); + } + } + } + } + } +} + +void qwen35_vision_rope_positions(int columns, int rows, int merge, std::vector & out) { + const size_t patches = size_t(columns) * rows; + out.resize(4 * patches); + size_t at = 0; + for (int by = 0; by < rows; by += merge) { + for (int bx = 0; bx < columns; bx += merge) { + for (int dy = 0; dy < merge; ++dy) { + for (int dx = 0; dx < merge; ++dx, ++at) { + out[at] = out[2 * patches + at] = by + dy; + out[patches + at] = out[3 * patches + at] = bx + dx; + } + } + } + } +} + +struct Qwen35VisionTower::Impl { + Qwen35VisionConfig config; + MmprojFile file; + ggml_backend_t backend = nullptr; + ggml_gallocr_t allocator = nullptr; + std::vector position_table; + std::vector blocks; + ggml_tensor * patch_first = nullptr, * patch_second = nullptr, * patch_bias = nullptr; + ggml_tensor * post_w = nullptr, * post_b = nullptr; + ggml_tensor * merge_in_w = nullptr, * merge_in_b = nullptr; + ggml_tensor * merge_out_w = nullptr, * merge_out_b = nullptr; + + ~Impl() { + if (allocator) ggml_gallocr_free(allocator); + } + + // Looks a tensor up and checks its shape; records the first failure. + ggml_tensor * need(const std::string & name, std::initializer_list shape, std::string & error) { + ggml_tensor * t = file.tensor(name); + bool ok = t != nullptr && shape.size() <= GGML_MAX_DIMS; + int axis = 0; + for (int64_t extent : shape) ok = ok && t->ne[axis++] == extent; + for (; ok && axis < GGML_MAX_DIMS; ++axis) ok = t->ne[axis] == 1; + if (!ok && error.empty()) error = "projector tensor is missing or has the wrong shape: " + name; + return ok ? t : nullptr; + } + + bool read_config(int language_dimension, std::string & error) { + uint32_t layers = 0, dimension = 0, heads = 0, intermediate = 0, patch = 0, merge = 0, projection = 0; + std::vector mean, deviation; + if (!file.u32("clip.vision.block_count", layers) || + !file.u32("clip.vision.embedding_length", dimension) || + !file.u32("clip.vision.attention.head_count", heads) || + !file.u32("clip.vision.feed_forward_length", intermediate) || + !file.u32("clip.vision.patch_size", patch) || + !file.u32("clip.vision.spatial_merge_size", merge) || + !file.u32("clip.vision.projection_dim", projection) || + !file.f32("clip.vision.attention.layer_norm_epsilon", config.epsilon) || + !file.f32_array("clip.vision.image_mean", mean) || mean.size() != 3 || + !file.f32_array("clip.vision.image_std", deviation) || deviation.size() != 3) { + error = "projector metadata is incomplete"; + return false; + } + // The patch reordering in encode() is written for 2x2 merging, and + // the vision rope splits each head into four equal sections. + if (layers == 0 || layers > 256 || heads == 0 || dimension == 0 || dimension > 16384 || + dimension % heads != 0 || (dimension / heads) % 4 != 0 || intermediate == 0 || + patch == 0 || patch > 64 || merge != 2) { + error = "projector geometry is not supported"; + return false; + } + if ((int) projection != language_dimension) { + error = "projector was built for a model of width " + std::to_string(projection) + + ", this model is " + std::to_string(language_dimension); + return false; + } + for (int i = 0; i < 3; ++i) { + if (!(deviation[i] > 0.0f)) { error = "projector image_std must be positive"; return false; } + config.mean[i] = mean[i]; + config.deviation[i] = deviation[i]; + } + config.layers = (int) layers; + config.dimension = (int) dimension; + config.heads = (int) heads; + config.intermediate = (int) intermediate; + config.patch_size = (int) patch; + config.merge = (int) merge; + config.language_dimension = language_dimension; + return true; + } + + bool bind_tensors(std::string & error) { + const int64_t d = config.dimension, ff = config.intermediate, p = config.patch_size; + const int64_t merged = d * config.merge * config.merge; + patch_first = need("v.patch_embd.weight", {p, p, 3, d}, error); + patch_second = need("v.patch_embd.weight.1", {p, p, 3, d}, error); + patch_bias = need("v.patch_embd.bias", {d}, error); + post_w = need("v.post_ln.weight", {d}, error); + post_b = need("v.post_ln.bias", {d}, error); + merge_in_w = need("mm.0.weight", {merged, merged}, error); + merge_in_b = need("mm.0.bias", {merged}, error); + merge_out_w = need("mm.2.weight", {merged, config.language_dimension}, error); + merge_out_b = need("mm.2.bias", {config.language_dimension}, error); + blocks.resize(config.layers); + for (int i = 0; i < config.layers; ++i) { + const std::string prefix = "v.blk." + std::to_string(i) + "."; + Block & b = blocks[i]; + b.ln1_w = need(prefix + "ln1.weight", {d}, error); + b.ln1_b = need(prefix + "ln1.bias", {d}, error); + b.qkv_w = need(prefix + "attn_qkv.weight", {d, 3 * d}, error); + b.qkv_b = need(prefix + "attn_qkv.bias", {3 * d}, error); + b.out_w = need(prefix + "attn_out.weight", {d, d}, error); + b.out_b = need(prefix + "attn_out.bias", {d}, error); + b.ln2_w = need(prefix + "ln2.weight", {d}, error); + b.ln2_b = need(prefix + "ln2.bias", {d}, error); + b.up_w = need(prefix + "ffn_up.weight", {d, ff}, error); + b.up_b = need(prefix + "ffn_up.bias", {ff}, error); + b.down_w = need(prefix + "ffn_down.weight", {ff, d}, error); + b.down_b = need(prefix + "ffn_down.bias", {d}, error); + } + if (!error.empty()) return false; + + ggml_tensor * table = file.tensor("v.position_embd.weight"); + const int side = table ? (int) std::lround(std::sqrt((double) table->ne[1])) : 0; + if (!table || table->type != GGML_TYPE_F32 || table->ne[0] != d || side < 2 || + (int64_t) side * side != table->ne[1]) { + error = "projector position table is missing or not a square F32 grid"; + return false; + } + config.position_side = side; + position_table.resize(size_t(side) * side * d); + ggml_backend_tensor_get(table, position_table.data(), 0, ggml_nbytes(table)); + return true; + } +}; + +Qwen35VisionTower::Qwen35VisionTower() : impl_(std::make_unique()) {} +Qwen35VisionTower::~Qwen35VisionTower() = default; + +const Qwen35VisionConfig & Qwen35VisionTower::config() const { return impl_->config; } +size_t Qwen35VisionTower::weight_bytes() const { return impl_->file.weight_bytes(); } + +bool Qwen35VisionTower::load(const std::string & path, ggml_backend_t backend, int language_dimension, + std::string & error) { + Impl & m = *impl_; + if (!m.file.load(path, backend, error)) return false; + if (m.file.projector_type() != PROJECTOR_TYPE) { + error = "projector type '" + m.file.projector_type() + "' is not " + PROJECTOR_TYPE; + return false; + } + if (m.file.has_tensor_prefix("v.deepstack.")) { + error = "projectors with deepstack branches are not supported"; + return false; + } + m.backend = backend; + return m.read_config(language_dimension, error) && m.bind_tensors(error); +} + +bool qwen35_vision_preprocess(const Qwen35VisionConfig & c, const DecodedRgb & image, + Qwen35Pixels & out, std::string & error) { + int width = 0, height = 0; + if (!qwen35_vision_target_size(c, (int) image.width, (int) image.height, width, height, error)) return false; + std::vector resized; + if (!resize_rgb_bicubic(image.pixels, (int) image.width, (int) image.height, width, height, resized, error)) { + return false; + } + out.width = width; + out.height = height; + out.grid_columns = width / (c.patch_size * c.merge); + out.grid_rows = height / (c.patch_size * c.merge); + const size_t plane = size_t(width) * height; + out.planar.resize(3 * plane); + for (int channel = 0; channel < 3; ++channel) { + float * dst = out.planar.data() + channel * plane; + for (size_t i = 0; i < plane; ++i) { + dst[i] = (resized[3 * i + channel] / 255.0f - c.mean[channel]) / c.deviation[channel]; + } + } + return true; +} + +bool Qwen35VisionTower::encode(const Qwen35Pixels & pixels, std::vector & rows, std::string & error) { + Impl & m = *impl_; + const Qwen35VisionConfig & c = m.config; + const int columns = pixels.width / c.patch_size, lines = pixels.height / c.patch_size; + const int64_t patches = int64_t(columns) * lines; + const int64_t tokens = pixels.tokens(); + if (!m.backend || tokens <= 0 || tokens > c.max_image_tokens || + columns != pixels.grid_columns * c.merge || lines != pixels.grid_rows * c.merge || + pixels.planar.size() != size_t(3) * pixels.width * pixels.height) { + error = "image was not prepared for this projector"; + return false; + } + + const int64_t d = c.dimension, head = d / c.heads; + ggml_init_params params{ggml_tensor_overhead() * GRAPH_NODES + ggml_graph_overhead_custom(GRAPH_NODES, false), + nullptr, /*no_alloc=*/true}; + std::unique_ptr owner(ggml_init(params)); + ggml_context * ctx = owner.get(); + if (!ctx) { error = "vision graph context allocation failed"; return false; } + ggml_cgraph * graph = ggml_new_graph_custom(ctx, GRAPH_NODES, false); + + ggml_tensor * image = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, pixels.width, pixels.height, 3, 1); + ggml_tensor * position_rows = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, d, patches); + ggml_tensor * rope_positions = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 4 * patches); + ggml_set_input(image); + ggml_set_input(position_rows); + ggml_set_input(rope_positions); + + // Patch embedding. The model was trained on two-frame clips; a still + // image is the same frame twice, so both temporal kernels see it. + const int p = c.patch_size; + ggml_tensor * x = ggml_add(ctx, ggml_conv_2d(ctx, m.patch_first, image, p, p, 0, 0, 1, 1), + ggml_conv_2d(ctx, m.patch_second, image, p, p, 0, 0, 1, 1)); + // [columns, lines, d] in raster order -> [d, patches] with the four + // patches of each 2x2 block adjacent, which is what the merger expects. + x = ggml_permute(ctx, x, 1, 2, 0, 3); + x = ggml_cont_4d(ctx, x, d * 2, columns / 2, lines, 1); + x = ggml_reshape_4d(ctx, x, d * 2, columns / 2, 2, lines / 2); + x = ggml_permute(ctx, x, 0, 2, 1, 3); + x = ggml_cont_2d(ctx, x, d, patches); + x = ggml_add(ctx, x, m.patch_bias); + x = ggml_add(ctx, x, position_rows); + + int sections[4] = {int(head / 4), int(head / 4), int(head / 4), int(head / 4)}; + const float attention_scale = 1.0f / std::sqrt(float(head)); + for (const Block & b : m.blocks) { + ggml_tensor * qkv = linear(ctx, b.qkv_w, b.qkv_b, layer_norm(ctx, x, b.ln1_w, b.ln1_b, c.epsilon)); + const auto part = [&](int index) { + return ggml_view_3d(ctx, qkv, head, c.heads, patches, ggml_row_size(qkv->type, head), qkv->nb[1], + ggml_row_size(qkv->type, d) * index); + }; + const auto rotate = [&](ggml_tensor * t) { + return ggml_rope_multi(ctx, t, rope_positions, nullptr, int(head / 2), sections, + GGML_ROPE_TYPE_VISION, 32768, 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); + }; + // Fused attention with half-precision keys and values, as the + // reference implementation runs it: the score matrix of a large + // image (3,900 patches squared per head) never touches memory. + ggml_tensor * q = ggml_permute(ctx, rotate(part(0)), 0, 2, 1, 3); + ggml_tensor * k = ggml_cast(ctx, ggml_permute(ctx, rotate(part(1)), 0, 2, 1, 3), GGML_TYPE_F16); + ggml_tensor * v = ggml_cast(ctx, ggml_permute(ctx, part(2), 0, 2, 1, 3), GGML_TYPE_F16); + ggml_tensor * mixed = ggml_flash_attn_ext(ctx, q, k, v, nullptr, attention_scale, 0.0f, 0.0f); + ggml_flash_attn_ext_set_prec(mixed, GGML_PREC_F32); + x = ggml_add(ctx, x, linear(ctx, b.out_w, b.out_b, ggml_reshape_2d(ctx, mixed, d, patches))); + + ggml_tensor * hidden = linear(ctx, b.up_w, b.up_b, layer_norm(ctx, x, b.ln2_w, b.ln2_b, c.epsilon)); + x = ggml_add(ctx, x, linear(ctx, b.down_w, b.down_b, ggml_gelu(ctx, hidden))); + } + + // Merger: each block of four patches becomes one language-model row. + x = layer_norm(ctx, x, m.post_w, m.post_b, c.epsilon); + x = ggml_reshape_2d(ctx, x, d * c.merge * c.merge, tokens); + x = linear(ctx, m.merge_out_w, m.merge_out_b, ggml_gelu(ctx, linear(ctx, m.merge_in_w, m.merge_in_b, x))); + ggml_set_output(x); + ggml_build_forward_expand(graph, x); + + if (!m.allocator) m.allocator = ggml_gallocr_new(ggml_backend_get_default_buffer_type(m.backend)); + if (!m.allocator || !ggml_gallocr_alloc_graph(m.allocator, graph)) { + error = "not enough device memory for the vision tower scratch"; + return false; + } + + std::vector positions; + std::vector rope; + qwen35_vision_position_rows(m.position_table, c.position_side, c.dimension, columns, lines, c.merge, positions); + qwen35_vision_rope_positions(columns, lines, c.merge, rope); + ggml_backend_tensor_set(image, pixels.planar.data(), 0, ggml_nbytes(image)); + ggml_backend_tensor_set(position_rows, positions.data(), 0, ggml_nbytes(position_rows)); + ggml_backend_tensor_set(rope_positions, rope.data(), 0, ggml_nbytes(rope_positions)); + + if (ggml_backend_graph_compute(m.backend, graph) != GGML_STATUS_SUCCESS) { + error = "vision tower compute failed"; + return false; + } + rows.resize(size_t(tokens) * c.language_dimension); + ggml_backend_tensor_get(x, rows.data(), 0, ggml_nbytes(x)); + return true; +} + +void Qwen35VisionTower::release_scratch() { + if (impl_->allocator) { + ggml_gallocr_free(impl_->allocator); + impl_->allocator = nullptr; + } +} + +} // namespace luce::vision diff --git a/server/src/qwen35/qwen35_vision.h b/server/src/qwen35/qwen35_vision.h new file mode 100644 index 000000000..bb5a9a389 --- /dev/null +++ b/server/src/qwen35/qwen35_vision.h @@ -0,0 +1,102 @@ +// Vision tower for Qwen3.5 / Qwen3.8 image input: the "qwen3vl_merger" +// projector published next to the model as an mmproj GGUF. Turns one image +// into rows of language-model embeddings, one per 2x2 block of patches. +// +// Split in two because the halves run on different threads: preprocessing is +// a CPU-only function of the config, safe while the GPU is busy or the tower +// is unloaded; encoding runs the tower graph and must be serialised with the +// rest of the backend's GPU work. +#pragma once + +#include "../common/vision/image_decode.h" + +#include "ggml-backend.h" + +#include +#include +#include +#include + +namespace luce::vision { + +struct Qwen35VisionConfig { + int layers = 0, dimension = 0, heads = 0, intermediate = 0; + int patch_size = 0, merge = 0; + int position_side = 0; // the learned position table is side x side + int language_dimension = 0; // width of one output row + float epsilon = 1e-6f; + float mean[3] = {0.5f, 0.5f, 0.5f}; + float deviation[3] = {0.5f, 0.5f, 0.5f}; + // Image tokens per image. The lower bound is the model's own; the upper + // bound keeps the tower's attention scratch near 1 GiB. + int min_image_tokens = 64; + int max_image_tokens = 1024; + + // Everything preprocessing and prompt expansion depend on. + bool same_geometry(const Qwen35VisionConfig & o) const { + return patch_size == o.patch_size && merge == o.merge && + language_dimension == o.language_dimension && + min_image_tokens == o.min_image_tokens && max_image_tokens == o.max_image_tokens && + mean[0] == o.mean[0] && mean[1] == o.mean[1] && mean[2] == o.mean[2] && + deviation[0] == o.deviation[0] && deviation[1] == o.deviation[1] && + deviation[2] == o.deviation[2]; + } +}; + +// A resized, normalised image. `planar` is channel major ([3, height, width]). +struct Qwen35Pixels { + int width = 0, height = 0; + int grid_columns = 0, grid_rows = 0; // image tokens across and down + std::vector planar; + + int tokens() const { return grid_columns * grid_rows; } +}; + +// Size the model resizes a width x height image to: both sides a multiple of +// patch_size * merge, area within the configured token bounds. +bool qwen35_vision_target_size(const Qwen35VisionConfig & config, int width, int height, + int & out_width, int & out_height, std::string & error); + +// Resizes the way the model was trained (bicubic, straight to the target +// size) and normalises. CPU only. +bool qwen35_vision_preprocess(const Qwen35VisionConfig & config, const DecodedRgb & image, + Qwen35Pixels & out, std::string & error); + +// Learned position rows for a columns x rows patch grid, bilinearly resampled +// from the side x side table with aligned corners and written in the tower's +// merged patch order. `table` and `out` hold `dimension` floats per position. +void qwen35_vision_position_rows(const std::vector & table, int side, int dimension, + int columns, int rows, int merge, std::vector & out); + +// Rotary positions of the patches, in merged order: [4 * patches] as the four +// consecutive axes (y, x, y, x) that ggml's vision rope expects. +void qwen35_vision_rope_positions(int columns, int rows, int merge, std::vector & out); + +class Qwen35VisionTower { +public: + Qwen35VisionTower(); + ~Qwen35VisionTower(); + Qwen35VisionTower(const Qwen35VisionTower &) = delete; + Qwen35VisionTower & operator=(const Qwen35VisionTower &) = delete; + + // `backend` must outlive the tower. `language_dimension` is the text + // model's hidden size; a projector built for another model is refused. + bool load(const std::string & path, ggml_backend_t backend, int language_dimension, + std::string & error); + + const Qwen35VisionConfig & config() const; + size_t weight_bytes() const; + + // Runs the tower. `rows` receives tokens() * language_dimension floats in + // reading order. One caller at a time. + bool encode(const Qwen35Pixels & pixels, std::vector & rows, std::string & error); + + // Frees the graph scratch between requests; the weights stay loaded. + void release_scratch(); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace luce::vision diff --git a/server/src/server/http_server.cpp b/server/src/server/http_server.cpp index d921380dc..18d8c7973 100644 --- a/server/src/server/http_server.cpp +++ b/server/src/server/http_server.cpp @@ -17,6 +17,7 @@ #endif #include "http_server.h" +#include "image_input.h" #include "engine/luce_engine.h" #include "admission.h" #include "common/concurrency/seq_engine.h" @@ -965,6 +966,7 @@ json build_props_body(const ServerConfig & config, {"reasoning_supported", reasoning_supported}, {"speculative_supported", speculative_supported}, {"tools_supported", tools_supported}, + {"image_input_supported", config.image_input_enabled}, }}, }; return body; @@ -1245,6 +1247,13 @@ HttpServer::HttpServer(luce::engine::LuceEngine & engine, config.disk_cache_continued_interval, config.disk_cache_cold_max_tokens}, backend_) { + config_.image_input_enabled = backend_.supports_images() && + config_.pflash_upstream_base.empty() && !backend_.seq_engine(); + if (backend_.supports_images() && !config_.image_input_enabled) { + std::fprintf(stderr, + "[server] WARNING: a vision projector is loaded but image input is off: it is " + "not available with upstream forwarding or concurrent sequence scheduling\n"); + } #ifdef LUCE_HAS_CURL curl_global_init(CURL_GLOBAL_DEFAULT); #endif @@ -2336,7 +2345,7 @@ bool HttpServer::validate_request_context( SocketHandle fd, const ParsedRequest & req, bool send_failure) { const int prompt_tokens = (int) req.prompt_tokens.size(); const bool pflash_will_run = - config_.pflash_mode != ServerConfig::PflashMode::OFF && + !req.images && config_.pflash_mode != ServerConfig::PflashMode::OFF && drafter_tokenizer_ != nullptr && (config_.pflash_mode == ServerConfig::PflashMode::ALWAYS || prompt_tokens >= config_.pflash_threshold); @@ -2429,6 +2438,26 @@ bool HttpServer::handle_model_request(SocketHandle fd, ParsedRequest & req, try { const json & body = req.raw_body; if (!parse_common_request_fields(fd, body, req)) return true; + // Image extraction and redaction apply only to an image-capable + // backend; every other backend sees the request exactly as before. + std::vector encoded_images; + if (config_.image_input_enabled) { + json normalized; + std::string extraction_error; + const ImageRequestPolicy image_policy{ + req.format == ApiFormat::OPENAI_CHAT, + config_.image_input_enabled, + backend_.image_placeholder()}; + if (!prepare_request_images(req.messages, image_policy, normalized, + encoded_images, extraction_error)) { + send_error(fd, 400, extraction_error); + return true; + } + req.messages = std::move(normalized); + redact_image_urls(req.raw_body); + redact_image_urls(req.messages); + } + const std::vector chat_messages = normalize_chat_messages(req.messages, req.format, tool_memory_); // Reasoning must be applied BEFORE rendering: the template injects @@ -2440,7 +2469,7 @@ bool HttpServer::handle_model_request(SocketHandle fd, ParsedRequest & req, // PPP rearrange (optional): peel ephemeral system banners into a // following system message so the first chat boundary is stable. std::vector render_messages = chat_messages; - if (config_.ppp_enabled && config_.ppp_rearrange && !req.tools.empty()) { + if (encoded_images.empty() && config_.ppp_enabled && config_.ppp_rearrange && !req.tools.empty()) { auto layout = PinFriendlyPrompt::rearrange(chat_messages, true); if (layout.rearranged) { render_messages = std::move(layout.messages); @@ -2463,6 +2492,14 @@ bool HttpServer::handle_model_request(SocketHandle fd, ParsedRequest & req, if (!render_and_tokenize_request(fd, render_messages, req)) return true; + std::string image_error; + if (!backend_.prepare_images(req.prompt_tokens, std::move(encoded_images), + uint64_t(std::max(0, config_.max_ctx)), uint64_t(std::max(0, req.max_output)), + req.images, image_error)) { + send_error(fd, 400, image_error); + return true; + } + // count_tokens: short-circuit after tokenization. Skip generation // entirely — Anthropic's contract is just {"input_tokens": N}. if (count_tokens_only) { @@ -2472,9 +2509,12 @@ bool HttpServer::handle_model_request(SocketHandle fd, ParsedRequest & req, send_response(fd, 200, "application/json", response.dump() + "\n"); return true; } - } catch (const std::exception & e) { + } catch (const json::parse_error & e) { send_error(fd, 400, std::string("JSON parse error: ") + e.what()); return true; + } catch (const std::exception & e) { + send_error(fd, 400, std::string("Invalid request: ") + e.what()); + return true; } if (!validate_request_context(fd, req, admission == nullptr)) { @@ -3266,6 +3306,15 @@ HttpServer::PreparedPrompt HttpServer::prepare_prompt( const ParsedRequest & req) { PreparedPrompt prepared; prepared.tokens = req.prompt_tokens; + if (req.images) { + if (!config_.image_input_enabled || !req.images->matches(prepared.tokens)) { + prepared.error_status = 400; + prepared.error = "image request binding or serving mode is invalid"; + } else { + prepared.images = req.images; + } + return prepared; + } if (config_.pflash_mode != ServerConfig::PflashMode::OFF && drafter_tokenizer_ != nullptr) { @@ -3322,6 +3371,7 @@ HttpServer::PreparedPrompt HttpServer::prepare_prompt( bool HttpServer::forward_upstream( ServerJob * job, const ParsedRequest & req, const PreparedPrompt & prepared) { + if (req.images) return false; #ifdef LUCE_HAS_CURL if (config_.pflash_upstream_base.empty()) return false; @@ -3397,6 +3447,7 @@ bool HttpServer::forward_upstream( HttpServer::GenerationCacheState HttpServer::prepare_generation_cache( const ParsedRequest & req, PreparedPrompt & prepared, GenerateRequest & generate_request) { + if (req.images) return {}; auto & effective_prompt = prepared.tokens; // Tool-heavy requests prefer the reusable system/tool boundary under eviction. const bool prefer_inline_snap = !req.tools.empty(); @@ -3799,6 +3850,7 @@ void HttpServer::finalize_generation_cache( GenerationCacheState & cache, const GenerateResult & result, int completion_tokens, bool visible_output_seen, bool client_disconnected) { + if (req.images) return; const auto & effective_prompt = prepared.tokens; const bool generation_produced_output = result.ok() && completion_tokens > 0 && visible_output_seen && !client_disconnected; @@ -3949,7 +4001,7 @@ void HttpServer::remember_agent_turn( for (const auto & call : emitter.tool_calls()) call_ids.push_back(call.id); tool_memory_.remember(call_ids, assistant_content); - if (!replay_cache) return; + if (!replay_cache || req.images) return; if (!config_.agent_turn_cache || prefix_cache_.disabled()) return; // Cache only stateless-equivalent prompts. Compression and token rewrites // need a separate replay contract. @@ -4049,6 +4101,8 @@ void HttpServer::prepare_generation_inputs( : req.max_output; inputs.request.prompt = prepared.tokens; + inputs.request.images = prepared.images; + inputs.request.force_ar_decode = bool(prepared.images); inputs.request.n_gen = inputs.generation_cap; inputs.request.sampler = req.sampler; inputs.request.do_sample = req.sampler.needs_logit_processing(); @@ -4241,7 +4295,9 @@ void HttpServer::process_job(ServerJob * job) { // Track live status for /status page. RAII guard ensures idle on all paths. std::string prompt_excerpt; - if (!req.prompt_tokens.empty()) { + if (req.images) { + prompt_excerpt = req.rendered_prompt.substr(0, 200); + } else if (!req.prompt_tokens.empty()) { // Decode first ~40 tokens as a prompt excerpt (cheap, bounded). const int excerpt_len = (std::min)((int)req.prompt_tokens.size(), 40); std::vector excerpt_toks(req.prompt_tokens.begin(), @@ -4521,7 +4577,7 @@ void HttpServer::process_job(ServerJob * job) { // Record performance for /status page. PerfRecord perf; - perf.prompt_tokens = (int)req.prompt_tokens.size(); + perf.prompt_tokens = effective_prompt_tokens; perf.completion_tokens = completion_tokens; // Use actual prefilled token count: on cache hit the backend only // prefills the delta beyond the cached prefix, so dividing the full diff --git a/server/src/server/http_server.h b/server/src/server/http_server.h index 044ffad37..c6a63b3bd 100644 --- a/server/src/server/http_server.h +++ b/server/src/server/http_server.h @@ -193,6 +193,7 @@ struct ServerConfig { int fa_window = 0; int ddtree_budget = 0; bool speculative_enabled = false; + bool image_input_enabled = false; bool target_sharding = false; // Prefill chunk size (bargs.chunk). Exposed at /props.runtime.chunk so // bench/snapshot tooling can capture the full server config — needed @@ -305,6 +306,7 @@ PflashQueryWindow find_pflash_query_window( struct ParsedRequest { ApiFormat format; std::vector prompt_tokens; // tokenized prompt + ImagePromptHandle images; std::string rendered_prompt; int max_output = 4096; bool stream = true; @@ -434,6 +436,7 @@ class HttpServer { struct PreparedPrompt { std::vector tokens; + ImagePromptHandle images; bool compressed = false; bool flowkv = false; int full_cache_served_tokens = -1; diff --git a/server/src/server/image_input.cpp b/server/src/server/image_input.cpp new file mode 100644 index 000000000..8f943a0cb --- /dev/null +++ b/server/src/server/image_input.cpp @@ -0,0 +1,236 @@ +#include "image_input.h" + +#include +#include +#include + +namespace luce::common { +namespace { +int base64_value(char c) { + if (c >= 'A' && c <= 'Z') return c - 'A'; + if (c >= 'a' && c <= 'z') return c - 'a' + 26; + if (c >= '0' && c <= '9') return c - '0' + 52; + if (c == '+') return 62; + if (c == '/') return 63; + return -1; +} +bool reserved_placeholder(std::string_view text, std::string_view placeholder) { + return text.find(placeholder) != std::string_view::npos; +} +void require(bool valid, const char * message) { + if (!valid) throw std::invalid_argument(message); +} +void validate_message_structure(const nlohmann::json & messages, std::string_view placeholder) { + std::vector> pending{{&messages, 0}}; + while (!pending.empty()) { + const auto [value, depth] = pending.back(); + pending.pop_back(); + require(depth <= 64, "image message nesting exceeds 64 levels"); + if (value->is_string()) { + require(!reserved_placeholder(value->get_ref(), placeholder), + "text contains the reserved image placeholder"); + } else if (value->is_structured()) { + for (const auto & child : *value) pending.emplace_back(&child, depth + 1); + } + } +} +} + +bool parse_image_data_url(std::string_view url, EncodedImage & image, + std::string & error, size_t max_bytes) { + image = {}; + error.clear(); + try { + EncodedImage parsed; + constexpr std::string_view jpeg = "data:image/jpeg;base64,"; + constexpr std::string_view png = "data:image/png;base64,"; + if (url.substr(0, jpeg.size()) == jpeg) { + parsed.mime_type = "image/jpeg"; + url.remove_prefix(jpeg.size()); + } else if (url.substr(0, png.size()) == png) { + parsed.mime_type = "image/png"; + url.remove_prefix(png.size()); + } else { + throw std::invalid_argument("image_url must be a JPEG or PNG base64 data URL"); + } + require(!url.empty() && url.size() % 4 == 0, "invalid image base64 length"); + size_t padding = url.back() == '=' ? 1 : 0; + if (url[url.size() - 2] == '=') ++padding; + const size_t decoded_bytes = url.size() / 4 * 3 - padding; + require(decoded_bytes > 0 && decoded_bytes <= max_bytes, "image exceeds encoded byte limit"); + for (size_t i = 0; i < url.size() - padding; ++i) { + require(base64_value(url[i]) >= 0, "invalid image base64 alphabet or padding"); + } + for (size_t i = url.size() - padding; i < url.size(); ++i) { + require(url[i] == '=', "invalid image base64 padding"); + } + if (padding == 1) require((base64_value(url[url.size() - 2]) & 3) == 0, "noncanonical image base64 padding"); + if (padding == 2) require((base64_value(url[url.size() - 3]) & 15) == 0, "noncanonical image base64 padding"); + parsed.bytes.reserve(decoded_bytes); + for (size_t i = 0; i < url.size(); i += 4) { + uint32_t value = 0; + for (size_t j = 0; j < 4; ++j) { + value = value << 6 | uint32_t(url[i+j] == '=' ? 0 : base64_value(url[i+j])); + } + for (int shift : {16, 8, 0}) { + if (parsed.bytes.size() < decoded_bytes) parsed.bytes.push_back(uint8_t(value >> shift)); + } + } + constexpr std::array jpeg_signature{255, 216, 255}; + constexpr std::array png_signature{137, 80, 78, 71, 13, 10, 26, 10}; + if (parsed.mime_type == "image/jpeg") { + require(parsed.bytes.size() >= jpeg_signature.size() && + std::equal(jpeg_signature.begin(), jpeg_signature.end(), parsed.bytes.begin()), + "image bytes do not match JPEG media type"); + } else { + require(parsed.bytes.size() >= png_signature.size() && + std::equal(png_signature.begin(), png_signature.end(), parsed.bytes.begin()), + "image bytes do not match PNG media type"); + } + image = std::move(parsed); + return true; + } catch (const std::exception & e) { + error = e.what(); + return false; + } +} + +bool extract_chat_images(const nlohmann::json & messages, std::string_view placeholder, + nlohmann::json & normalized, + std::vector & images, + std::string & error, const ImageInputLimits & limits) { + normalized = nullptr; + images.clear(); + error.clear(); + try { + require(!placeholder.empty(), "the backend names no image placeholder"); + require(messages.is_array(), "image messages must be an array"); + validate_message_structure(messages, placeholder); + nlohmann::json result = messages; + std::vector collected; + size_t total_bytes = 0; + for (auto & message : result) { + require(message.is_object(), "each image message must be an object"); + if (!message.contains("content")) continue; + auto & content = message["content"]; + if (!content.is_array()) continue; + std::string text_segment; + for (auto & part : content) { + require(part.is_object(), "each content part must be an object"); + const auto type = part.value("type", std::string()); + if (type == "text" || type == "input_text" || type == "output_text") { + text_segment += part.value("text", std::string()); + continue; + } + require(type != "image" && type != "input_image", "use image_url content parts for images"); + if (type != "image_url") continue; + require(!reserved_placeholder(text_segment, placeholder), "text contains the reserved image placeholder"); + text_segment.clear(); + require(message.value("role", std::string("user")) == "user", "images are supported only in user messages"); + require(collected.size() < limits.image_count, "too many images in request"); + require(part.contains("image_url") && part["image_url"].is_object() && + part["image_url"].contains("url") && part["image_url"]["url"].is_string(), + "image_url must contain a string url"); + const auto & descriptor = part["image_url"]; + if (descriptor.contains("detail")) { + require(descriptor["detail"].is_string(), "image detail must be auto, low, or high"); + const auto detail = descriptor["detail"].get(); + require(detail == "auto" || detail == "low" || detail == "high", "image detail must be auto, low, or high"); + } + require(total_bytes <= limits.request_bytes, "images exceed request byte limit"); + EncodedImage decoded; + std::string decode_error; + const size_t remaining = std::min(limits.image_bytes, limits.request_bytes - total_bytes); + if (!parse_image_data_url(descriptor["url"].get_ref(), decoded, decode_error, remaining)) { + error = std::move(decode_error); + return false; + } + total_bytes += decoded.bytes.size(); + collected.push_back(std::move(decoded)); + part = {{"type", "text"}, {"text", std::string(placeholder)}}; + } + require(!reserved_placeholder(text_segment, placeholder), "text contains the reserved image placeholder"); + } + normalized = std::move(result); + images = std::move(collected); + return true; + } catch (const nlohmann::json::exception &) { + error = "invalid image content field type"; + return false; + } catch (const std::exception & e) { + error = e.what(); + return false; + } +} + +static bool contains_image_content(const nlohmann::json & value) { + std::vector pending{&value}; + while (!pending.empty()) { + const auto & current = *pending.back(); + pending.pop_back(); + if (current.is_object()) { + auto type = current.find("type"); + if (type != current.end() && type->is_string() && + (*type == "image" || *type == "input_image" || *type == "image_url")) return true; + for (const auto & item : current.items()) { + if (item.key() == "image_url") return true; + pending.push_back(&item.value()); + } + } else if (current.is_array()) { + for (const auto & item : current) pending.push_back(&item); + } + } + return false; +} + +bool prepare_request_images(const nlohmann::json & messages, + const ImageRequestPolicy & policy, + nlohmann::json & normalized, + std::vector & images, + std::string & error, const ImageInputLimits & limits) { + normalized = nullptr; + images.clear(); + error.clear(); + // A backend without image input keeps its existing handling of image + // parts: a client whose history holds an image must not start failing. + if (!policy.image_capable) { + normalized = messages; + return true; + } + const bool has_images = contains_image_content(messages); + if (has_images && !policy.chat_completions) { + error = "image input is supported only through /v1/chat/completions image_url parts"; + return false; + } + if (policy.chat_completions) { + nlohmann::json prepared; + std::vector extracted; + if (!extract_chat_images(messages, policy.placeholder, prepared, extracted, error, limits)) return false; + if (contains_image_content(prepared)) { + error = "images must be user content-array image_url parts"; + return false; + } + normalized = std::move(prepared); + images = std::move(extracted); + } else { + normalized = messages; + } + return true; +} + +void redact_image_urls(nlohmann::json & value) { + std::vector pending{&value}; + while (!pending.empty()) { + auto * current = pending.back(); + pending.pop_back(); + if (current->is_object()) { + for (auto & item : current->items()) { + if (item.key() == "image_url") item.value() = "[image omitted]"; + else pending.push_back(&item.value()); + } + } else if (current->is_array()) { + for (auto & item : *current) pending.push_back(&item); + } + } +} +} // namespace luce::common diff --git a/server/src/server/image_input.h b/server/src/server/image_input.h new file mode 100644 index 000000000..16c770de6 --- /dev/null +++ b/server/src/server/image_input.h @@ -0,0 +1,47 @@ +#pragma once + +#include "common/image_prompt.h" + +#include +#include +#include +#include +#include +#include + +namespace luce::common { + + +inline constexpr size_t MAX_IMAGE_BYTES = 16 * 1024 * 1024; + +struct ImageInputLimits { + size_t image_bytes = MAX_IMAGE_BYTES; + size_t request_bytes = 32 * 1024 * 1024; + size_t image_count = 4; +}; + +struct ImageRequestPolicy { + bool chat_completions = false; + bool image_capable = false; + // Text that stands for one image in the rendered prompt; the backend's + // chat template maps it to the model's image marker. Users may not send it. + std::string placeholder; +}; + +bool parse_image_data_url(std::string_view url, EncodedImage & image, + std::string & error, size_t max_bytes = MAX_IMAGE_BYTES); +bool extract_chat_images(const nlohmann::json & messages, + std::string_view placeholder, + nlohmann::json & normalized, + std::vector & images, + std::string & error, + const ImageInputLimits & limits = {}); +bool prepare_request_images(const nlohmann::json & messages, + const ImageRequestPolicy & policy, + nlohmann::json & normalized, + std::vector & images, + std::string & error, + const ImageInputLimits & limits = {}); +void redact_image_urls(nlohmann::json & value); + +} // namespace luce::common diff --git a/server/src/server/server_main.cpp b/server/src/server/server_main.cpp index 30202f9d3..a70e05081 100644 --- a/server/src/server/server_main.cpp +++ b/server/src/server/server_main.cpp @@ -85,6 +85,7 @@ static void print_usage(const char * prog) { " Defaults to the first block; request model names\n" " do not change generation routing.\n" " --draft Draft model for speculative decode\n" + " --mmproj Vision projector GGUF: enables image input (Qwen3.5/3.8, DS4V)\n" " --port Listen port (default: 8080)\n" " --host Bind address (default: 0.0.0.0)\n" " --max-ctx Max context length (default: 131072)\n" @@ -358,6 +359,12 @@ static int parse_model_options(int argc, char ** argv, ModelOptions & model, } if (std::strcmp(argv[i], "--draft") == 0 && i + 1 < argc) { bargs.draft_path = argv[++i]; + } else if (std::strcmp(argv[i], "--mmproj") == 0) { + if (i + 1 >= argc) { + std::fprintf(stderr, "[server] --mmproj needs a projector GGUF path\n"); + return 2; + } + bargs.mmproj_path = argv[++i]; } else if (std::strcmp(argv[i], "--port") == 0 && i + 1 < argc) { sconfig.port = std::atoi(argv[++i]); } else if (std::strcmp(argv[i], "--host") == 0 && i + 1 < argc) { diff --git a/server/test/test_ds4_mix_fitter.cpp b/server/test/test_ds4_mix_fitter.cpp new file mode 100644 index 000000000..bef9793e0 --- /dev/null +++ b/server/test/test_ds4_mix_fitter.cpp @@ -0,0 +1,75 @@ +#define main ds4_mix_converter_main +#include "../tools/ds4_mix_converter/ds4_mix_converter.cpp" +#undef main + +int main(int argc, char ** argv) { + try { + for (float value : {0.0f, -1.0f, 1.0f}) { + HistogramFitter fitter; + float values[16]; + std::fill_n(values, 16, value); + fitter.add_half(values, nullptr); + for (int levels : {4, 8}) { + const auto books = fitter.fit(levels); + if (books != fitter.fit(levels)) fail("fit is nondeterministic"); + for (int population = 0; population < 2; ++population) { + float previous = -std::numeric_limits::infinity(); + for (int i = 0; i < levels; ++i) { + float v = bf16_to_float(books[population*levels+i]); + if (!std::isfinite(v) || v < -1 || v > 1 || v <= previous) + fail("codebook is not finite, bounded, and strictly ordered"); + previous = v; + } + } + float x[32] = {}; + block_rocmfp2 q2; + block_rocmfp3 q3; + if (levels == 4 && !rocmfpx_quantize_row_fp2_mix_ref(x, &q2, 32, books.data(), nullptr)) + fail("repaired fp2 codebook rejected by encoder"); + if (levels == 8 && !rocmfpx_quantize_row_fp3_mix_ref(x, &q3, 32, books.data(), nullptr)) + fail("repaired fp3 codebook rejected by encoder"); + } + } + bool repaired = false; + const auto close = HistogramFitter::round_centers({-1.0f, 0.5f, 0.5001f, 1.0f}, repaired); + if (!repaired || bf16_to_float(close[2]) <= bf16_to_float(close[1])) + fail("BF16 rounding collision was not repaired"); + repaired = false; + const auto distinct = HistogramFitter::round_centers({-1.0f, -0.5f, 0.5f, 1.0f}, repaired); + if (repaired || distinct != std::vector{0xbf80, 0xbf00, 0x3f00, 0x3f80}) + fail("nondegenerate centers changed"); + HistogramFitter stamped; + float zero[16] = {}; + stamped.add_half(zero, nullptr); + std::vector stamps; + stamped.fit(4, "layer=42 expert=164 down", &stamps); + if (stamps.size() != 2 || stamps[0].find("layer=42 expert=164 down") == std::string::npos) + fail("repair stamp missing expert identity"); + bool rejected = false; + try { HistogramFitter().fit(4); } catch (const std::runtime_error &) { rejected = true; } + if (!rejected) fail("empty histogram accepted"); + if (argc != 1 && argc != 5) fail("usage: test_ds4_mix_fitter [SOURCE IMATRIX LAYER EXPERT]"); + if (argc == 5) { + SafeTensorSet source(argv[1]); + std::optional imatrix = load_imatrix(argv[2]); + const int layer = parse_nonnegative(argv[3], "layer"); + const int expert = parse_nonnegative(argv[4], "expert"); + HistogramFitter gate_up, down; + for (const auto & recipe : kExpertRecipes) { + const auto shape = validate_expert_source(source, layer, expert, recipe); + const auto * importance = require_imatrix(imatrix, target_expert_name(layer, recipe), shape.in); + add_expert_to_fitter(source, layer, expert, recipe, importance, + recipe.books == BookSource::GateUpJoint ? gate_up : down); + } + const std::string label = "layer=" + std::to_string(layer) + " expert=" + std::to_string(expert); + gate_up.fit(kGuLevels, label + " gate_up"); + down.fit(kP4Levels, label + " down"); + std::cout << "PASS: source replay " << label << "\n"; + } + std::cout << "PASS: degenerate fitter, BF16 ordering, determinism, codec acceptance, empty rejection\n"; + return 0; + } catch (const std::exception & e) { + std::cerr << "FAIL: " << e.what() << "\n"; + return 1; + } +} diff --git a/server/test/test_ds4v_image_assembly.cpp b/server/test/test_ds4v_image_assembly.cpp new file mode 100644 index 000000000..91fbf4830 --- /dev/null +++ b/server/test/test_ds4v_image_assembly.cpp @@ -0,0 +1,145 @@ +#include "deepseek4/deepseek4_image_assembly.h" +#include +#include +#include + +using namespace luce::vision; +static void check(bool ok, const char * message) { + if (!ok) throw std::runtime_error(message); +} +static PromptImage fixture(uint64_t position) { + PromptImage image; + image.input.plan.aligner_rows = 2; + image.input.plan.aligner_cols = 2; + image.layout.span = {position, position + 1, position + 9, position + 10}; + image.layout.types = {ImageTokenType::Pad, ImageTokenType::Start, + ImageTokenType::Image, ImageTokenType::Image, ImageTokenType::Newline, + ImageTokenType::Image, ImageTokenType::Image, ImageTokenType::Pad, + ImageTokenType::End, ImageTokenType::Pad}; + image.layout.permutation = {2, 0, 3, 1}; + return image; +} + +int main() { + try { + ImageRequestGate gate; + auto lease = gate.try_acquire(); + check(bool(lease) && !gate.try_acquire(), "concurrent image request was admitted"); + auto retained_lease = lease; + lease.reset(); + check(!gate.try_acquire(), "request copy released image admission early"); + retained_lease.reset(); + lease = gate.try_acquire(); + check(bool(lease), "completed request did not release image admission"); + { + ImageRequestGate transient; + retained_lease = transient.try_acquire(); + } + retained_lease.reset(); + const ImageSentinels sentinels{{10,11}, {20,21}, {30,31}, {40,41}}; + const ImageRaster raster{4, 2, {100,101,200,201,300,301,400,401}}; + const std::vector expected{20,21,10,11,300,301,100,101,30,31, + 400,401,200,201,20,21,40,41,20,21}; + std::string error; + std::vector output{999}; + check(assemble_image_rows(fixture(1).layout, raster, sentinels, 2, output, error), "assembly rejected"); + check(output == expected, "sentinel identity or raster permutation applied incorrectly"); + auto bad_assembly = [&](ImageLayout layout, ImageRaster values, ImageSentinels marks) { + output = {999}; + check(!assemble_image_rows(layout, values, marks, 2, output, error), "malformed assembly accepted"); + check(output == std::vector{999} && !error.empty(), "assembly failure changed output"); + }; + auto layout = fixture(1).layout; + layout.permutation = {2,0,2,1}; bad_assembly(layout,raster,sentinels); + layout = fixture(1).layout; layout.permutation[0] = -1; bad_assembly(layout,raster,sentinels); + layout = fixture(1).layout; layout.permutation[0] = 4; bad_assembly(layout,raster,sentinels); + layout = fixture(1).layout; layout.types[4] = static_cast(99); bad_assembly(layout,raster,sentinels); + layout = fixture(1).layout; layout.types[1] = ImageTokenType::End; bad_assembly(layout,raster,sentinels); + layout = fixture(1).layout; layout.span.visible_end--; bad_assembly(layout,raster,sentinels); + auto values = raster; values.columns = 3; bad_assembly(fixture(1).layout,values,sentinels); + values = raster; values.values.pop_back(); bad_assembly(fixture(1).layout,values,sentinels); + values = raster; values.values[0] = std::numeric_limits::quiet_NaN(); bad_assembly(fixture(1).layout,values,sentinels); + auto marks = sentinels; marks.pad[0] = std::numeric_limits::infinity(); bad_assembly(fixture(1).layout,raster,marks); + marks = sentinels; marks.end.pop_back(); bad_assembly(fixture(1).layout,raster,marks); + + std::vector images{fixture(1),fixture(12)}; + const ImageRows old{{777},{888}}; + ImageRows result = old; + int calls = 0; + ImageEncode fail_second = [&](const PromptImage &, ImageRaster & out, std::string & reason) { + if (++calls == 2) { reason = "second image failed"; return false; } + out = raster; return true; + }; + check(!materialize_image_rows(images,sentinels,2,fail_second,{},result,error) && + calls == 2 && result == old && error == "second image failed", "second-image failure is not atomic"); + calls = 0; + bool stop = false; + ImageEncode cancel_after_first = [&](const PromptImage &, ImageRaster & out, std::string &) { + ++calls; out = raster; stop = true; return true; + }; + check(!materialize_image_rows(images,sentinels,2,cancel_after_first,[&] { return stop; },result,error) && + calls == 1 && result == old && !error.empty(), "cancellation launched a remaining encode"); + calls = 0; + check(!materialize_image_rows(images,sentinels,2,cancel_after_first,[] { return true; },result,error) && + calls == 0 && result == old, "pre-cancelled materialization launched encode"); + ImageEncode good = [&](const PromptImage &, ImageRaster & out, std::string &) { + out = raster; + for (float & v : out.values) v += 1000.0f * calls; + ++calls; return true; + }; + check(materialize_image_rows(images,sentinels,2,good,{},result,error) && calls == 2, + "valid materialization failed"); + check(result[0] == expected && result[1][4] == 1300 && result[1][0] == 20, + "equal-layout images lost distinct raster values"); + calls = 0; + auto wrong_plan = images; wrong_plan[1].input.plan.aligner_cols = 3; + ImageRows retained = result; + check(!materialize_image_rows(wrong_plan,sentinels,2,good,{},result,error) && calls == 0 && + result == retained, "bad second-image shape was not rejected before encoding"); + check(!materialize_image_rows(images,sentinels,2, + [](const PromptImage &, ImageRaster &, std::string &) -> bool { throw 1; },{},result,error) && + result == retained, "callback exception changed output"); + + PreparedImagePrompt prompt; + prompt.images = images; + prompt.tokens.assign(23, 7); + for (const auto & image : images) { + for (size_t r = 0; r < image.layout.types.size(); ++r) + prompt.tokens[size_t(image.layout.span.block_begin) + r] = 100 + int32_t(image.layout.types[r]); + } + size_t text_calls = 0, text_tokens = 0; + TextEmbed embed = [&](const int32_t * ids, size_t count, float * out) { + ++text_calls; text_tokens += count; + for (size_t i = 0; i < count; ++i) { + check(ids[i] >= 0 && ids[i] < 100, "external ID reached ordinary embedder"); + out[2*i] = float(ids[i]); out[2*i+1] = float(ids[i]+1); + } + return true; + }; + check(embed_image_prompt_chunk(prompt,result,100,2,0,23,embed,output,error), "mixed prompt failed"); + check(text_calls == 3 && text_tokens == 3 && output.size() == 46 && + output[0] == 7 && output[6] == 300 && output[28] == 1300 && output[44] == 7, + "mixed embedding order or offsets are wrong"); + check(embed_image_prompt_chunk(prompt,result,100,2,1,10,{},output,error) && output == expected, + "exact image block slice failed"); + auto reject_chunk = [&](const PreparedImagePrompt & p, size_t start, size_t count) { + output = {999}; text_calls = 0; + check(!embed_image_prompt_chunk(p,result,100,2,start,count,embed,output,error) && + output == std::vector{999} && text_calls == 0 && !error.empty(), + "invalid chunk reached embedder or changed output"); + }; + reject_chunk(prompt,2,9); reject_chunk(prompt,0,5); + auto invalid = prompt; invalid.tokens[0] = 100; reject_chunk(invalid,0,23); + invalid = prompt; invalid.tokens[3] = 100; reject_chunk(invalid,0,23); + invalid = prompt; invalid.tokens[22] = -1; reject_chunk(invalid,0,23); + output = {999}; + check(!embed_image_prompt_chunk(prompt,result,100,2,0,23, + [](const int32_t *,size_t,float *) { return false; },output,error) && + output == std::vector{999}, "text callback failure changed output"); + std::cout << "PASS image assembly identities/permutation, atomic materialization/cancellation, mixed embedding boundaries\n"; + return 0; + } catch (const std::exception & e) { + std::cerr << "FAIL: " << e.what() << '\n'; + return 1; + } +} diff --git a/server/test/test_ds4v_image_integration.cpp b/server/test/test_ds4v_image_integration.cpp new file mode 100644 index 000000000..45cb7967f --- /dev/null +++ b/server/test/test_ds4v_image_integration.cpp @@ -0,0 +1,229 @@ +#include "deepseek4_image_budget.h" +#include "deepseek4_image_spans.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +using luce::vision::ImageSpanView; +using luce::vision::TokenSpan; +using luce::vision::atomic_image_chunk; +using luce::vision::image_block_at; +using luce::vision::remaining_expert_budget; +using luce::vision::valid_image_spans; +constexpr uint64_t MAX = std::numeric_limits::max(); +size_t checks = 0; + +void require(bool condition, const std::string & description) { + ++checks; + if (!condition) throw std::runtime_error(description); +} + +ImageSpanView view(const std::vector & spans) { + return {spans.data(), spans.size()}; +} + +bool bisects(const std::vector & spans, uint64_t point) { + for (const auto & span : spans) { + if (span.block_begin < point && point < span.block_end) return true; + } + return false; +} + +// Enumerate every legal endpoint, independently of the helper's interval walk. +// A small preferred chunk may expand only when no positive legal endpoint fits. +int oracle(const std::vector & spans, uint64_t position, + int preferred, uint64_t remaining, int capacity) { + if (preferred <= 0 || capacity <= 0 || uint64_t(preferred) > remaining || + remaining > MAX - position || bisects(spans, position)) return 0; + int below = 0; + int above = 0; + for (int length = 1; length <= capacity && uint64_t(length) <= remaining; ++length) { + if (bisects(spans, position + uint64_t(length))) continue; + if (length <= preferred) below = length; + else if (!above) above = length; + } + return below ? below : above; +} + +void check_oracle(const std::vector & spans, uint64_t position, + int preferred, uint64_t remaining, int capacity) { + const int expected = oracle(spans, position, preferred, remaining, capacity); + const int actual = atomic_image_chunk(view(spans), position, preferred, remaining, capacity); + require(actual == expected, "endpoint oracle mismatch at position=" + std::to_string(position) + + " preferred=" + std::to_string(preferred) + " capacity=" + std::to_string(capacity)); + if (actual) { + require(actual > 0 && actual <= capacity && uint64_t(actual) <= remaining, + "chunk must make bounded positive progress"); + require(!bisects(spans, position + uint64_t(actual)), "chunk endpoint splits an image block"); + } +} + +void validation_and_lookup() { + require(valid_image_spans({}, 0), "empty prompt/view is valid"); + require(!image_block_at({}, 0), "empty lookup is null"); + require(!valid_image_spans({nullptr, 1}, 100), "nonempty null view rejected"); + std::array too_many{}; + require(!valid_image_spans({too_many.data(), too_many.size()}, 100), "more than four images rejected"); + const std::vector spans{{10, 13, 18, 20}, {20, 20, 25, 25}, {30, 31, 33, 35}}; + require(valid_image_spans(view(spans), 35), "adjacent and separated blocks valid"); + require(!image_block_at(view(spans), 9), "text before block excluded"); + require(image_block_at(view(spans), 10) == &spans[0], "leading padding belongs to image block"); + require(image_block_at(view(spans), 12) == &spans[0], "leading padding is not ordinary text"); + require(image_block_at(view(spans), 13) == &spans[0], "visible begin included"); + require(image_block_at(view(spans), 18) == &spans[0], "trailing padding belongs to block"); + require(image_block_at(view(spans), 20) == &spans[1], "adjacent boundary belongs to next block"); + require(!image_block_at(view(spans), 25), "half-open end excluded"); + require(!image_block_at(view(spans), 29), "gap excluded"); + require(!image_block_at(view(spans), 35), "final end excluded"); + for (const auto & invalid : std::vector>{ + {{9, 8, 10, 11}}, {{9, 9, 9, 11}}, {{9, 9, 12, 11}}, + {{9, 9, 10, 36}}, {{0, 0, 1, 385}}, {{5, 5, 7, 9}, {8, 8, 10, 12}}, + {{10, 10, 11, 12}, {1, 1, 2, 3}}, {{MAX, 0, 1, 2}}}) { + require(!valid_image_spans(view(invalid), 35), "malformed/overlapping/out-of-range span rejected"); + } + const std::vector largest{{0, 3, 380, 384}}; + require(valid_image_spans(view(largest), 384), "384-token block admitted"); + const std::vector overlong{{0, 3, 380, 385}}; + require(!valid_image_spans(view(overlong), 385), "385-token block rejected even inside prompt"); + const std::vector upper{{MAX - 384, MAX - 381, MAX - 1, MAX}}; + require(valid_image_spans(view(upper), MAX), "valid upper-limit span does not overflow"); + require(atomic_image_chunk(view(upper), MAX - 384, 1, 384, 1024) == 384, + "upper-limit atomic extension reaches exact end"); + require(atomic_image_chunk({}, MAX - 3, 1, 4, 1024) == 0, "position plus remaining overflow rejected"); + require(atomic_image_chunk({}, 0, 0, 10, 1024) == 0, "zero proposed chunk rejected"); + require(atomic_image_chunk({}, 0, -1, 10, 1024) == 0, "negative proposed chunk rejected"); + require(atomic_image_chunk({}, 0, 1, 10, 0) == 0, "zero capacity rejected"); + require(atomic_image_chunk({}, 0, 1, 10, -1) == 0, "negative capacity rejected"); + require(atomic_image_chunk({}, 0, 11, 10, 1024) == 0, "caller must clamp preferred chunk to remaining"); +} + +void production_boundary_matrix() { + const std::vector spans{ + {100, 103, 479, 484}, {484, 484, 500, 500}, + {32640, 32644, 33020, 33024}, {33024, 33024, 33031, 33032}}; + constexpr uint64_t prompt = 35000; + require(valid_image_spans(view(spans), prompt), "four-image fixture valid"); + for (int preferred : {1, 128, 512, 1024, 4096}) { + for (uint64_t position : {uint64_t(0), uint64_t(99), uint64_t(100), uint64_t(101), + uint64_t(483), uint64_t(484), uint64_t(499), uint64_t(500), uint64_t(32639), + uint64_t(32640), uint64_t(32768), uint64_t(33023), uint64_t(33024), uint64_t(33032)}) { + check_oracle(spans, position, preferred, prompt - position, 1024); + } + uint64_t position = 0; + size_t steps = 0; + while (position < prompt) { + const int proposed = int(std::min(uint64_t(preferred), prompt - position)); + const int count = atomic_image_chunk(view(spans), position, proposed, prompt - position, 1024); + require(count > 0, "complete traversal must not stall when every block fits capacity"); + require(!bisects(spans, position + uint64_t(count)), "complete traversal preserves atomic blocks"); + require(count <= 1024, "preferred 4096 cannot exceed independent hard capacity 1024"); + position += uint64_t(count); + require(++steps <= prompt, "traversal terminates"); + } + require(position == prompt, "complete traversal consumes exact prompt"); + } + require(atomic_image_chunk(view(spans), 32640, 128, prompt - 32640, 1024) == 384, + "image crossing 32768 extends past context boundary atomically"); + require(atomic_image_chunk(view(spans), 100, 1, prompt - 100, 383) == 0, + "capacity smaller than complete block rejected"); + require(atomic_image_chunk(view(spans), 100, 1, 383, 1024) == 0, + "remaining prompt smaller than complete block rejected"); + require(atomic_image_chunk(view(spans), 99, 128, prompt - 99, 128) == 1, + "ordinary prefix remains consumable before oversized image"); + for (int proposed : {100, 484, 500, 1024}) check_oracle(spans, 0, proposed, prompt, 1024); +} + +void exhaustive_endpoints() { + constexpr uint64_t prompt = 14; + // All ordered pairs of blocks, including adjacency, varying image lengths, + // all positions, and capacities that can both fit and reject a whole image. + for (uint64_t a = 0; a < prompt; ++a) { + for (uint64_t b = a + 1; b <= std::min(prompt, a + 4); ++b) { + for (uint64_t c = b; c < prompt; ++c) { + for (uint64_t d = c + 1; d <= std::min(prompt, c + 4); ++d) { + const std::vector spans{{a, a, b, b}, {c, c, d, d}}; + require(valid_image_spans(view(spans), prompt), "generated spans valid"); + for (uint64_t position = 0; position <= prompt; ++position) { + for (int capacity : {1, 2, 3, 4, 7, 16}) { + for (int preferred : {1, 2, 4, 8, 16}) { + const uint64_t remaining = prompt - position; + check_oracle(spans, position, preferred, remaining, capacity); + } + } + } + } + } + } + } +} + +void memory_budget() { + constexpr uint64_t MiB = 1024 * 1024; + constexpr uint64_t GiB = 1024 * MiB; + constexpr uint64_t vision = luce::vision::SCRATCH_RESERVATION; + constexpr uint64_t workspace = 76 * MiB; + require(vision == 2 * GiB, "retain qualified two-GiB vision reservation"); + const uint64_t initial = remaining_expert_budget(24 * GiB, 4 * GiB, GiB, 256 * MiB, 512 * MiB, vision); + require(initial == 16640 * MiB, "combined fixed charges leave expected expert budget"); + // A free-memory snapshot turns every completed allocation into core usage. + // Preloading the projector must be charged exactly once in either order. + constexpr uint64_t projector = 700 * MiB; + const uint64_t before_projector = remaining_expert_budget( + 24 * GiB, 4 * GiB, GiB, 256 * MiB, 512 * MiB, vision + projector); + const uint64_t after_projector = remaining_expert_budget( + 24 * GiB, 4 * GiB + projector, GiB, 256 * MiB, 512 * MiB, vision); + require(before_projector == after_projector, "preloaded projector charged once regardless of load order"); + const uint64_t after_workspace = remaining_expert_budget( + 24 * GiB, 4 * GiB + workspace, GiB, 256 * MiB, 512 * MiB, vision, workspace); + require(initial == after_workspace, "resident workspace offsets only its included reservation"); + require(remaining_expert_budget(24 * GiB, 4 * GiB + workspace, GiB, 256 * MiB, + 512 * MiB, vision) == initial - workspace, + "unproven workspace residency cannot receive a credit"); + require(remaining_expert_budget(MAX, 0, 0, 0, 0, vision, vision + 1) == 0, + "workspace credit cannot exceed reservation"); + require(remaining_expert_budget(100, 20, 20, 20, 20, 20) == 0, "exact exhaustion returns zero"); + require(remaining_expert_budget(99, 20, 20, 20, 20, 20) == 0, "one-byte shortage fails closed"); + require(remaining_expert_budget(MAX, MAX, 1, 0, 0, 0) == 0, "overflowing sum cannot wrap into budget"); + require(remaining_expert_budget(MAX, 1, MAX, 0, 0, 0) == 0, "late exhaustion cannot wrap"); + require(remaining_expert_budget(MAX, 0, 0, 0, 0, 0) == MAX, "uncharged maximum preserved"); + require(remaining_expert_budget(MAX, MAX - 7, 2, 1, 1, 2) == 1, "near-maximum arithmetic remains exact"); + // Independent conservation law for uniform expert rounds: tighter primary + // headroom must transfer the same quantized storage amount to the cold owner. + constexpr uint64_t round = 64 * MiB; + constexpr uint64_t total_experts = 32 * GiB; + uint64_t last_hot = initial / round * round; + uint64_t last_cold = total_experts - last_hot; + for (uint64_t extra = 0; extra <= 4 * GiB; extra += 17 * MiB) { + const uint64_t budget = remaining_expert_budget(24 * GiB, 4 * GiB, GiB, + 256 * MiB, 512 * MiB, vision + extra); + const uint64_t hot = budget / round * round; + const uint64_t cold = total_experts - hot; + require(hot <= last_hot && cold >= last_cold, "reduced hot budget cannot reduce cold demand"); + require(hot + cold == total_experts && budget - hot < round, "placement conserves expert bytes"); + require(last_hot - hot == cold - last_cold, "lost hot bytes equal additional cold bytes"); + last_hot = hot; + last_cold = cold; + } +} +} // namespace + +int main() { + try { + validation_and_lookup(); + production_boundary_matrix(); + exhaustive_endpoints(); + memory_budget(); + std::cout << "PASS: DS4 image integration invariants checks=" << checks << '\n'; + return EXIT_SUCCESS; + } catch (const std::exception & error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return EXIT_FAILURE; + } +} diff --git a/server/test/test_ds4v_image_policy.cpp b/server/test/test_ds4v_image_policy.cpp new file mode 100644 index 000000000..ff0e1268b --- /dev/null +++ b/server/test/test_ds4v_image_policy.cpp @@ -0,0 +1,61 @@ +#include "deepseek4/deepseek4_image_policy.h" +#include +#include +#include +#include +#include + +using namespace luce::vision; +static void check(bool ok, const char * reason) { if (!ok) throw std::runtime_error(reason); } +static bool visible(int64_t q, int64_t k, int64_t w, int64_t begin=-1, int64_t end=-1) { + bool result = false; + check(raw_key_visible(q,k,w,begin,end,result), "valid visibility rejected"); + return result; +} +int main() { + try { + ImageExpertSelection result; + std::string error; + float scores[] = {1,2,3,4}, bias[] = {0,8,0,0}; + check(select_image_experts(scores,bias,4,2,result,error), "valid selection rejected"); + check(result.count==2 && result.indices[0]==1 && result.indices[1]==3, "bias selection"); + check(result.weights[0]==0.5f && result.weights[1]==1.0f, "unbiased weights"); + float ties[] = {3,2,1,0}; + check(select_image_experts(scores,ties,4,4,result,error), "ties rejected"); + for (int i=0;i<4;++i) check(result.indices[i]==i, "lower-index tie handling"); + auto reject = [&](const float * s,const float * b,size_t n,size_t k,float scale=1.5f) { + result.count=9; + check(!select_image_experts(s,b,n,k,result,error,scale), "invalid selection accepted"); + check(result.count==0 && !error.empty(), "failure result/error contract"); + }; + reject(nullptr,bias,4,2); reject(scores,nullptr,4,2); + reject(scores,bias,0,0); reject(scores,bias,257,2); + reject(scores,bias,4,0); reject(scores,bias,4,5); + reject(scores,bias,4,2,0); reject(scores,bias,4,2,-1); + reject(scores,bias,4,2,std::numeric_limits::infinity()); + float invalid[] = {-1,2,3,4}; reject(invalid,bias,4,2); + invalid[0]=std::numeric_limits::quiet_NaN(); reject(invalid,bias,4,2); + invalid[0]=std::numeric_limits::infinity(); reject(scores,invalid,4,2); + float huge[] = {std::numeric_limits::max(),1,1,1}; reject(huge,huge,4,2); + float zero[] = {0,0,0,0}; reject(zero,bias,4,2); + float sum_overflow[] = {std::numeric_limits::max(),std::numeric_limits::max(),0,0}; + reject(sum_overflow,zero,4,2); + check(visible(0,0,1) && !visible(0,1,1), "initial causal row"); + check(visible(10,7,4) && !visible(10,6,4) && !visible(10,11,4), "window boundaries"); + check(visible(10,20,4,8,21) && visible(10,7,4,8,21), "image union"); + check(visible(20,8,4,8,21) && !visible(20,7,4,8,21), "long image"); + check(!visible(7,8,4,8,21) && !visible(21,8,4,8,21), "outside image isolation"); + check(!visible(10,30,4,8,21), "other future image isolation"); + const int64_t max=std::numeric_limits::max(); + check(visible(max,max,1) && visible(max,max-2,3) && !visible(max,max-3,3), "near integer limit"); + check(visible(max-2,max-1,3,max-3,max), "near limit image"); + bool value=true; + for (auto args : {std::array{-1,0,1,-1,-1}, {0,-1,1,-1,-1}, + {0,0,0,-1,-1}, {0,0,1,-1,2}, {0,0,1,2,2}, {0,0,1,3,2}}) { + check(!raw_key_visible(args[0],args[1],args[2],args[3],args[4],value) && !value, + "invalid visibility accepted"); + } + std::cout << "PASS routing selection/weights/ties, invalid contracts, raw visibility boundaries and int64 limits\n"; + return 0; + } catch (const std::exception & e) { std::cerr << "FAIL: " << e.what() << '\n'; return 1; } +} diff --git a/server/test/test_ds4v_image_prompt.cpp b/server/test/test_ds4v_image_prompt.cpp new file mode 100644 index 000000000..f24901a53 --- /dev/null +++ b/server/test/test_ds4v_image_prompt.cpp @@ -0,0 +1,97 @@ +#include "deepseek4_image_prompt.h" +#include +#include +#include + +using namespace luce::vision; +constexpr int32_t marker=129264,vocab=129280; +static void check(bool value,const char * why) { if (!value) throw std::runtime_error(why); } +static ImagePatchInput small_image(uint16_t pixel=0) { + ImagePatchInput image; + image.plan={42,42,3,3,1,1,false}; + image.patches_bf16.assign(9*588,pixel); + return image; +} +static void rejected(const PreparedImagePrompt & result,ImagePromptError expected) { + check(!result && result.error==expected,"wrong failure category"); + check(result.tokens.empty() && result.images.empty() && !result.message.empty(),"nontransactional failure"); +} +int main() { + try { + auto image=small_image(); + auto text=prepare_image_prompt({1,2,42},{ }); + check(bool(text) && text.tokens==std::vector({1,2,42}) && text.images.empty(),"text path"); + const std::vector body={ImageTokenType::Start,ImageTokenType::Image, + ImageTokenType::Pad,ImageTokenType::Newline,ImageTokenType::Pad,ImageTokenType::End}; + for (uint64_t start=0;start<4;++start) { + std::vector input(start,42); input.push_back(marker); input.push_back(7); + auto result=prepare_image_prompt(input,{image}); + check(bool(result) && result.images.size()==1,"single image"); + const auto & layout=result.images[0].layout; + std::vector expected(3-start,ImageTokenType::Pad); + expected.insert(expected.end(),body.begin(),body.end()); + check(layout.types==expected && layout.permutation==std::vector({0}),"source grounded tiny layout"); + check(layout.span.block_begin==start && layout.span.visible_begin==3 && + layout.span.visible_end==9 && layout.span.block_end==9,"span residues"); + check(result.tokens.size()==10 && result.tokens.back()==7,"expanded count/text tail"); + for (size_t i=0;i(expected[i]),"generated IDs"); + } + auto pair=prepare_image_prompt({marker,marker},{image,small_image(0x3f80)}); + check(bool(pair) && pair.tokens.size()==17 && pair.images[1].layout.span.block_begin==9 && + pair.images[1].layout.span.visible_begin==11 && pair.images[1].layout.span.block_end==17,"consecutive expanded positions"); + auto spaced=prepare_image_prompt({marker,55,marker,66},{image,image}); + check(bool(spaced) && spaced.images[1].layout.span.block_begin==10 && spaced.tokens[9]==55 && + spaced.tokens.back()==66,"text between images"); + auto copy=pair; + pair.images[0].input.patches_bf16[0]=0x3f80; + check(copy.images[0].input.patches_bf16[0]==0,"copy owns patches"); + auto moved=std::move(copy); + check(moved.images[1].input.patches_bf16[0]==0x3f80 && moved.tokens.size()==17,"move lifetime"); + auto independent=prepare_image_prompt({marker},{image}); + image.patches_bf16[0]=0xbf80; + check(independent.images[0].input.patches_bf16[0]==0,"input lifetime independence"); + auto other_request=prepare_image_prompt({marker},{small_image(0x3f80)}); + check(independent.tokens==other_request.tokens,"equal layout fixture"); + independent.images[0].input.patches_bf16[0]=0xbf80; + check(other_request.images[0].input.patches_bf16[0]==0x3f80,"distinct requests own image storage"); + check(bool(prepare_image_prompt(std::vector(4,marker),std::vector(4,image))),"four-image boundary"); + rejected(prepare_image_prompt({marker},{}),ImagePromptError::MarkerCount); + rejected(prepare_image_prompt({1},{image}),ImagePromptError::MarkerCount); + rejected(prepare_image_prompt({marker,marker},{image}),ImagePromptError::MarkerCount); + rejected(prepare_image_prompt({-1},{}),ImagePromptError::InvalidToken); + for (int kind=0;kind<5;++kind) rejected(prepare_image_prompt({vocab+kind},{}),ImagePromptError::InvalidToken); + rejected(prepare_image_prompt({marker},{image},{},{129279,marker}),ImagePromptError::InvalidContract); + rejected(prepare_image_prompt({marker},{image},{},{vocab,marker-1}),ImagePromptError::InvalidContract); + rejected(prepare_image_prompt(std::vector(5,marker),std::vector(5,image)),ImagePromptError::ImageCount); + auto bad=image; bad.plan.resized_width++; + rejected(prepare_image_prompt({marker,marker},{image,bad}),ImagePromptError::InvalidPlan); + bad=image; bad.plan.aligner_rows=2; + rejected(prepare_image_prompt({marker},{bad}),ImagePromptError::InvalidPlan); + bad=image; bad.plan.vit_rows=0; + rejected(prepare_image_prompt({marker},{bad}),ImagePromptError::InvalidPlan); + bad=image; bad.plan.vit_rows=std::numeric_limits::max(); + rejected(prepare_image_prompt({marker},{bad}),ImagePromptError::InvalidPlan); + bad=image; bad.plan={1008,672,48,72,16,24,false}; + rejected(prepare_image_prompt({marker},{bad}),ImagePromptError::InvalidPlan); + bad=image; bad.patches_bf16.pop_back(); + rejected(prepare_image_prompt({marker},{bad}),ImagePromptError::InvalidPatches); + for (uint16_t word : {uint16_t(0x3f81),uint16_t(0xbf81),uint16_t(0x7f80),uint16_t(0x7fc1)}) { + bad=image; bad.patches_bf16[0]=word; + rejected(prepare_image_prompt({marker},{bad}),ImagePromptError::InvalidPatches); + } + check(bool(prepare_image_prompt({marker},{image},{10,1,9})),"exact context fit"); + rejected(prepare_image_prompt({marker},{image},{9,1,9}),ImagePromptError::ContextOverflow); + rejected(prepare_image_prompt({marker},{image},{10,1,8}),ImagePromptError::TokenLimit); + rejected(prepare_image_prompt({1},{},{0,0,1}),ImagePromptError::InvalidLimits); + rejected(prepare_image_prompt({1},{},{1,2,1}),ImagePromptError::InvalidLimits); + rejected(prepare_image_prompt({1},{},{1,0,0}),ImagePromptError::InvalidLimits); + rejected(prepare_image_prompt({1},{},{2147483648ULL,0,1}),ImagePromptError::InvalidLimits); + rejected(prepare_image_prompt({1},{},{2147483647,0,2147483647}),ImagePromptError::InvalidLimits); + rejected(prepare_image_prompt({1},{},{2147483647,2147483647,1}),ImagePromptError::ContextOverflow); + check(bool(prepare_image_prompt({1},{},{2147483647,2147483646,1})),"wide exact arithmetic"); + rejected(prepare_image_prompt({1},{},{1,std::numeric_limits::max(),1}),ImagePromptError::InvalidLimits); + std::cout<<"PASS text, cardinality, source layout residues, ordering, ownership, negative contracts, checked context\n"; + return 0; + } catch (const std::exception & e) { std::cerr<<"FAIL: "< +#include +#include +#include + +#define CHECK(cond, msg) REQUIRE(cond) + +namespace { + +static_assert(sizeof(block_rocmfp2) == 10, "qtype-106 ABI changed"); +static_assert(sizeof(block_rocmfp3) == 14, "qtype-105 ABI changed"); + +struct Ds4MixConverterFixture : CommonFixture { + using CommonFixture::CommonFixture; +}; + +static uint16_t f32_to_bf16_bits(float x) { + uint32_t u = 0; + std::memcpy(&u, &x, 4); + if (!std::isfinite(x)) { + return (uint16_t) (u >> 16); + } + const uint32_t rounded = u + 0x7FFFu + ((u >> 16) & 1u); + return (uint16_t) (rounded >> 16); +} + +static float bf16_bits_to_f32(uint16_t h) { + const uint32_t u = (uint32_t) h << 16; + float x = 0.0f; + std::memcpy(&x, &u, 4); + return x; +} + +static void fill_books2(uint16_t out[8]) { + const float b0[4] = {-2.0f, -0.5f, 0.5f, 2.0f}; + const float b1[4] = {-4.0f, -1.0f, 1.0f, 4.0f}; + for (int i = 0; i < 4; ++i) { + out[i] = f32_to_bf16_bits(b0[i]); + out[4 + i] = f32_to_bf16_bits(b1[i]); + } +} + +static void fill_books3(uint16_t out[16]) { + const float b0[8] = {-4.0f, -2.0f, -1.0f, -0.5f, 0.5f, 1.0f, 2.0f, 4.0f}; + const float b1[8] = {-8.0f, -4.0f, -2.0f, -1.0f, 1.0f, 2.0f, 4.0f, 8.0f}; + for (int i = 0; i < 8; ++i) { + out[i] = f32_to_bf16_bits(b0[i]); + out[8 + i] = f32_to_bf16_bits(b1[i]); + } +} + +static float row_max_err(const float * a, const float * b, int n) { + float m = 0.0f; + for (int i = 0; i < n; ++i) { + const float d = std::fabs(a[i] - b[i]); + if (d > m) m = d; + } + return m; +} + +TEST_CASE(Ds4MixConverterFixture, fp2_mix_roundtrip) { + float x[32]; + for (int i = 0; i < 32; ++i) { + x[i] = -4.0f + (float) i * (8.0f / 31.0f); + } + uint16_t books[8]; + fill_books2(books); + block_rocmfp2 q[1]; + float y[32]; + CHECK(rocmfpx_quantize_row_fp2_mix_ref(x, q, 32, books, nullptr), + "fp2 mix quantize accepts valid books without imatrix"); + rocmfpx_dequantize_row_fp2_mix(q, y, 32, books); + CHECK(row_max_err(x, y, 32) < 2.0f, "fp2 mix roundtrip error stays bounded"); +} + +TEST_CASE(Ds4MixConverterFixture, fp3_mix_roundtrip) { + float x[32]; + for (int i = 0; i < 32; ++i) { + x[i] = -4.0f + (float) i * (8.0f / 31.0f); + } + uint16_t books[16]; + fill_books3(books); + block_rocmfp3 q[1]; + float y[32]; + CHECK(rocmfpx_quantize_row_fp3_mix_ref(x, q, 32, books, nullptr), + "fp3 mix quantize accepts valid books without imatrix"); + rocmfpx_dequantize_row_fp3_mix(q, y, 32, books); + CHECK(row_max_err(x, y, 32) < 1.0f, "fp3 mix roundtrip error stays bounded"); +} + +TEST_CASE(Ds4MixConverterFixture, fp2_byte_layout_reference) { + float x[32]; + for (int i = 0; i < 32; ++i) { + x[i] = (i % 2) ? 0.21f : -0.21f; + } + uint16_t books[8]; + fill_books2(books); + block_rocmfp2 q[1]; + float y[32]; + CHECK(rocmfpx_quantize_row_fp2_mix_ref(x, q, 32, books, nullptr), + "fp2 mix quantize succeeds for layout check"); + rocmfpx_dequantize_row_fp2_mix(q, y, 32, books); + float lvl[2][4]; + for (int b = 0; b < 2; ++b) { + for (int i = 0; i < 4; ++i) { + lvl[b][i] = bf16_bits_to_f32(books[b * 4 + i]); + } + } + for (int i = 0; i < 32; ++i) { + const uint8_t meta = q[0].e[i >= 16]; + const int book = meta >> 7; + const float scale = rocmfpx_ue4m3_to_fp32(meta & 0x7f); + const int code = (q[0].qs[i >> 2] >> (2 * (i & 3))) & 3; + const float expect = scale * lvl[book][code]; + CHECK(y[i] == expect, "fp2 decoded value matches the byte layout exactly"); + } +} + +TEST_CASE(Ds4MixConverterFixture, weighted_encode_honors_imatrix) { + float x[32]; + float w[32]; + for (int i = 0; i < 32; ++i) { + x[i] = 0.05f; + w[i] = 0.0f; + } + x[0] = 1.9f; + w[0] = 1000000.0f; + uint16_t books[8]; + fill_books2(books); + block_rocmfp2 q[1]; + float y[32]; + CHECK(rocmfpx_quantize_row_fp2_mix_ref(x, q, 32, books, w), + "fp2 mix quantize accepts an imatrix"); + rocmfpx_dequantize_row_fp2_mix(q, y, 32, books); + CHECK(std::fabs(y[0] - 1.9f) < 0.5f, "weighted encode protects the important value"); +} + +TEST_CASE(Ds4MixConverterFixture, malformed_books_rejected) { + float x[32] = {0.0f}; + block_rocmfp2 q2[1]; + block_rocmfp3 q3[1]; + uint16_t unsorted[8]; + fill_books2(unsorted); + unsorted[1] = unsorted[0]; + CHECK(!rocmfpx_quantize_row_fp2_mix_ref(x, q2, 32, unsorted, nullptr), + "fp2 mix rejects an unsorted codebook"); + uint16_t nanbook[8]; + fill_books2(nanbook); + nanbook[2] = 0x7FC0u; + CHECK(!rocmfpx_quantize_row_fp2_mix_ref(x, q2, 32, nanbook, nullptr), + "fp2 mix rejects a non-finite codebook level"); + CHECK(!rocmfpx_quantize_row_fp2_mix_ref(x, q2, 32, nullptr, nullptr), + "fp2 mix rejects null codebooks"); + uint16_t unsorted3[16]; + fill_books3(unsorted3); + unsorted3[9] = unsorted3[8]; + CHECK(!rocmfpx_quantize_row_fp3_mix_ref(x, q3, 32, unsorted3, nullptr), + "fp3 mix rejects an unsorted codebook"); +} + +} // namespace diff --git a/server/test/test_gpu_page_pool.cpp b/server/test/test_gpu_page_pool.cpp new file mode 100644 index 000000000..65c0e4c2c --- /dev/null +++ b/server/test/test_gpu_page_pool.cpp @@ -0,0 +1,61 @@ +// The estimate of GPU driver pages that MemAvailable does not count. +#include "common/gpu_page_pool.h" + +#include + +using luce::common::reclaimable_gpu_page_pool_bytes; + +static int failures = 0; +static void check(bool condition, const char * message) { + if (!condition) { std::fprintf(stderr, "FAIL: %s\n", message); ++failures; } +} + +int main() { + constexpr uint64_t GiB = 1ULL << 30; + // 100,000,000 kB total, 31,500,000 kB attributed -> 68,500,000 kB unattributed. + const char * meminfo = + "MemTotal: 100000000 kB\n" + "MemFree: 10000000 kB\n" + "MemAvailable: 28000000 kB\n" + "Buffers: 0 kB\n" + "Cached: 20000000 kB\n" + "AnonPages: 1000000 kB\n" + "Shmem: 300000 kB\n" // already inside Cached, must not be counted twice + "Slab: 500000 kB\n" + "HugePages_Total: 0\n" + "Hugepagesize: 2048 kB\n"; + const uint64_t unattributed = 68500000ULL * 1024; + + check(reclaimable_gpu_page_pool_bytes(meminfo, 0) == unattributed - GiB, + "idle GPU: everything unattributed, less the margin for other drivers"); + check(reclaimable_gpu_page_pool_bytes(meminfo, 5 * GiB) == unattributed - 6 * GiB, + "live GPU buffers are in use, not reclaimable"); + check(reclaimable_gpu_page_pool_bytes(meminfo, unattributed) == 0, + "all unattributed memory is live GPU buffers"); + check(reclaimable_gpu_page_pool_bytes(meminfo, unattributed + 9 * GiB) == 0, "never negative"); + + const char * huge = + "MemTotal: 100000000 kB\n" + "MemFree: 10000000 kB\n" + "HugePages_Total: 20000\n" + "Hugepagesize: 2048 kB\n"; // 40,960,000 kB of huge pages are attributed + check(reclaimable_gpu_page_pool_bytes(huge, 0) == (100000000ULL - 10000000 - 40960000) * 1024 - GiB, + "huge pages count as attributed"); + + const char * mixed_huge = + "MemTotal: 100000000 kB\n" + "MemFree: 10000000 kB\n" + "HugePages_Total: 20000\n" + "Hugepagesize: 2048 kB\n" + "Hugetlb: 50000000 kB\n"; // includes a pool of another page size + check(reclaimable_gpu_page_pool_bytes(mixed_huge, 0) == (100000000ULL - 10000000 - 50000000) * 1024 - GiB, + "Hugetlb covers huge page pools of every size"); + + check(reclaimable_gpu_page_pool_bytes("MemFree: 5 kB\n", 0) == 0, "no MemTotal, no estimate"); + check(reclaimable_gpu_page_pool_bytes("", 0) == 0 && reclaimable_gpu_page_pool_bytes(nullptr, 0) == 0, + "empty input"); + + if (failures) { std::fprintf(stderr, "%d failure(s)\n", failures); return 1; } + std::printf("OK\n"); + return 0; +} diff --git a/server/test/test_image_decode.cpp b/server/test/test_image_decode.cpp new file mode 100644 index 000000000..e09ac3f39 --- /dev/null +++ b/server/test/test_image_decode.cpp @@ -0,0 +1,151 @@ +// JPEG/PNG decoding shared by every vision model. +#include "common/vision/image_decode.h" + +#include +#include +#include + +using namespace luce::vision; + +static int failures = 0; +static void check(bool condition, const char * message) { + if (!condition) { std::fprintf(stderr, "FAIL: %s\n", message); ++failures; } +} + +// 3x2 PNG: red, green, blue / (10,20,30), (200,100,50), white. +const std::vector PNG_3X2 = { + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 3, + 0, 0, 0, 2, 8, 2, 0, 0, 0, 18, 22, 241, 77, 0, 0, 0, 24, 73, 68, 65, + 84, 120, 156, 99, 248, 207, 192, 192, 0, 193, 92, 34, 114, 39, 82, 140, 254, 255, 255, 15, + 0, 60, 25, 7, 149, 239, 194, 198, 216, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, + 130, +}; + +// 16x8 JPEG filled with (40,120,200). +const std::vector JPEG_16X8 = { + 255, 216, 255, 224, 0, 16, 74, 70, 73, 70, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, + 255, 219, 0, 67, 0, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 2, + 4, 3, 2, 2, 2, 2, 5, 4, 4, 3, 4, 6, 5, 6, 6, 6, 5, 6, 6, 6, + 7, 9, 8, 6, 7, 9, 7, 6, 6, 8, 11, 8, 9, 10, 10, 10, 10, 10, 6, 8, + 11, 12, 11, 10, 12, 9, 10, 10, 10, 255, 219, 0, 67, 1, 2, 2, 2, 2, 2, 2, + 5, 3, 3, 5, 10, 7, 6, 7, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 255, 192, + 0, 17, 8, 0, 8, 0, 16, 3, 1, 34, 0, 2, 17, 1, 3, 17, 1, 255, 196, 0, + 31, 0, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 255, 196, 0, 181, 16, 0, 2, 1, 3, 3, + 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 125, 1, 2, 3, 0, 4, 17, 5, 18, 33, + 49, 65, 6, 19, 81, 97, 7, 34, 113, 20, 50, 129, 145, 161, 8, 35, 66, 177, 193, 21, + 82, 209, 240, 36, 51, 98, 114, 130, 9, 10, 22, 23, 24, 25, 26, 37, 38, 39, 40, 41, + 42, 52, 53, 54, 55, 56, 57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, + 87, 88, 89, 90, 99, 100, 101, 102, 103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121, 122, + 131, 132, 133, 134, 135, 136, 137, 138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, + 165, 166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186, 194, 195, 196, 197, 198, + 199, 200, 201, 202, 210, 211, 212, 213, 214, 215, 216, 217, 218, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 255, 196, 0, 31, 1, 0, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 255, 196, 0, 181, 17, 0, 2, 1, 2, 4, 4, 3, 4, 7, + 5, 4, 4, 0, 1, 2, 119, 0, 1, 2, 3, 17, 4, 5, 33, 49, 6, 18, 65, 81, + 7, 97, 113, 19, 34, 50, 129, 8, 20, 66, 145, 161, 177, 193, 9, 35, 51, 82, 240, 21, + 98, 114, 209, 10, 22, 36, 52, 225, 37, 241, 23, 24, 25, 26, 38, 39, 40, 41, 42, 53, + 54, 55, 56, 57, 58, 67, 68, 69, 70, 71, 72, 73, 74, 83, 84, 85, 86, 87, 88, 89, + 90, 99, 100, 101, 102, 103, 104, 105, 106, 115, 116, 117, 118, 119, 120, 121, 122, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, 165, 166, + 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186, 194, 195, 196, 197, 198, 199, 200, + 201, 202, 210, 211, 212, 213, 214, 215, 216, 217, 218, 226, 227, 228, 229, 230, 231, 232, 233, 234, + 242, 243, 244, 245, 246, 247, 248, 249, 250, 255, 218, 0, 12, 3, 1, 0, 2, 17, 3, 17, + 0, 63, 0, 242, 58, 40, 162, 191, 181, 15, 228, 51, 255, 217, +}; + +// 8x8 greyscale JPEG filled with 128. +const std::vector JPEG_GREY_8X8 = { + 255, 216, 255, 224, 0, 16, 74, 70, 73, 70, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, + 255, 219, 0, 67, 0, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 2, + 4, 3, 2, 2, 2, 2, 5, 4, 4, 3, 4, 6, 5, 6, 6, 6, 5, 6, 6, 6, + 7, 9, 8, 6, 7, 9, 7, 6, 6, 8, 11, 8, 9, 10, 10, 10, 10, 10, 6, 8, + 11, 12, 11, 10, 12, 9, 10, 10, 10, 255, 192, 0, 11, 8, 0, 8, 0, 8, 1, 1, + 17, 0, 255, 196, 0, 31, 0, 0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 255, 196, 0, 181, 16, + 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 125, 1, 2, 3, 0, + 4, 17, 5, 18, 33, 49, 65, 6, 19, 81, 97, 7, 34, 113, 20, 50, 129, 145, 161, 8, + 35, 66, 177, 193, 21, 82, 209, 240, 36, 51, 98, 114, 130, 9, 10, 22, 23, 24, 25, 26, + 37, 38, 39, 40, 41, 42, 52, 53, 54, 55, 56, 57, 58, 67, 68, 69, 70, 71, 72, 73, + 74, 83, 84, 85, 86, 87, 88, 89, 90, 99, 100, 101, 102, 103, 104, 105, 106, 115, 116, 117, + 118, 119, 120, 121, 122, 131, 132, 133, 134, 135, 136, 137, 138, 146, 147, 148, 149, 150, 151, 152, + 153, 154, 162, 163, 164, 165, 166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186, + 194, 195, 196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214, 215, 216, 217, 218, 225, 226, + 227, 228, 229, 230, 231, 232, 233, 234, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 255, 218, + 0, 8, 1, 1, 0, 0, 63, 0, 43, 255, 217, +}; + +// 2x1 16-bit greyscale PNG with samples 0x8000 and 0x0100. +const std::vector PNG_GREY16_2X1 = { + 137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 13, 73, 72, 68, 82, 0, 0, 0, 2, 0, 0, 0, 1, 16, 0, 0, 0, 0, 129, 217, 252, 21, 0, 0, 0, 13, 73, 68, 65, 84, 120, 156, 99, 104, 96, 96, 100, 0, 0, 2, 7, 0, 130, 159, 82, 239, 216, 0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130, +}; + +static DecodeResult decode(const std::vector & bytes, const DecodeLimits & limits = {}) { + return decode_image({bytes.data(), bytes.size()}, limits); +} + +static bool near(int value, int expected) { return std::abs(value - expected) <= 3; } + +int main() { + { + const auto result = decode(PNG_3X2); + check(bool(result) && result.image.width == 3 && result.image.height == 2, "PNG dimensions"); + const std::vector expected = { + 255, 0, 0, 0, 255, 0, 0, 0, 255, 10, 20, 30, 200, 100, 50, 255, 255, 255}; + check(result.image.pixels == expected, "PNG pixels are exact row-major RGB"); + check(result.image.view().size == expected.size(), "view covers the pixels"); + } + { + const auto result = decode(JPEG_16X8); + check(bool(result) && result.image.width == 16 && result.image.height == 8, "JPEG dimensions"); + bool close = result.image.pixels.size() == 16u * 8u * 3u; + for (size_t i = 0; close && i < result.image.pixels.size(); i += 3) { + close = near(result.image.pixels[i], 40) && near(result.image.pixels[i + 1], 120) && + near(result.image.pixels[i + 2], 200); + } + check(close, "JPEG pixels match the source colour"); + } + { + const auto result = decode(JPEG_GREY_8X8); + bool grey = bool(result) && result.image.pixels.size() == 8u * 8u * 3u; + for (size_t i = 0; grey && i < result.image.pixels.size(); ++i) grey = near(result.image.pixels[i], 128); + check(grey, "greyscale JPEG expands to RGB"); + } + { + const auto result = decode(PNG_GREY16_2X1); + const std::vector expected = {128, 128, 128, 1, 1, 1}; + check(bool(result) && result.image.pixels == expected, "16-bit greyscale keeps its high byte"); + } + { + check(decode_image({nullptr, 0}).status.code == DecodeError::EmptyInput, "empty input"); + const std::vector text = {'n', 'o', 't', ' ', 'a', 'n', ' ', 'i', 'm', 'a', 'g', 'e'}; + check(decode(text).status.code == DecodeError::UnsupportedFormat, "unknown format"); + auto truncated = PNG_3X2; + truncated.resize(truncated.size() / 2); + const auto broken = decode(truncated); + check(!broken && broken.image.pixels.empty(), "truncated PNG fails without partial output"); + auto cut = JPEG_16X8; + cut.resize(cut.size() / 3); + check(!decode(cut), "truncated JPEG fails"); + } + { + DecodeLimits limits; + limits.max_encoded_bytes = PNG_3X2.size() - 1; + check(decode(PNG_3X2, limits).status.code == DecodeError::EncodedTooLarge, "encoded byte limit"); + limits = {}; + limits.max_decoded_pixels = 5; + check(decode(PNG_3X2, limits).status.code == DecodeError::DecodedTooLarge, "pixel count limit"); + limits = {}; + limits.max_dimension = 2; + check(decode(PNG_3X2, limits).status.code == DecodeError::DecodedTooLarge, "dimension limit"); + limits = {}; + limits.max_decoded_pixels = 6; + check(bool(decode(PNG_3X2, limits)), "exact pixel limit is accepted"); + } + if (failures) { std::fprintf(stderr, "%d failure(s)\n", failures); return 1; } + std::printf("OK\n"); + return 0; +} diff --git a/server/test/test_image_input.cpp b/server/test/test_image_input.cpp new file mode 100644 index 000000000..66b7d1bfa --- /dev/null +++ b/server/test/test_image_input.cpp @@ -0,0 +1,105 @@ +#include "server/image_input.h" + +#include +#include + +using namespace luce::common; +using json = nlohmann::json; + +// Any text works; the transport must not know a model's marker. +static constexpr char IMAGE_PLACEHOLDER[] = ""; + +static void check(bool condition, const char * message) { + if (!condition) throw std::runtime_error(message); +} +static json image_part(const std::string & url) { + return {{"type", "image_url"}, {"image_url", {{"url", url}}}}; +} +static json text_part(const std::string & text) { + return {{"type", "text"}, {"text", text}}; +} + +int main() { + try { + const std::string jpeg = "data:image/jpeg;base64,/9j/"; + const std::string png = "data:image/png;base64,iVBORw0KGgo="; + EncodedImage image; + std::string error; + check(parse_image_data_url(jpeg, image, error), "valid JPEG transport rejected"); + check(image.mime_type == "image/jpeg" && image.bytes == std::vector({255,216,255}), "JPEG bytes differ"); + check(parse_image_data_url(png, image, error), "valid PNG transport rejected"); + check(image.mime_type == "image/png" && image.bytes == std::vector({137,80,78,71,13,10,26,10}), "PNG bytes differ"); + for (const auto & bad : std::vector{ + "https://example.com/a.png", "file:///etc/passwd", "data:image/gif;base64,R0lG", + "data:image/png,iVBORw0KGgo=", "data:image/png;base64,", "data:image/png;base64,AAA", + "data:image/png;base64,AAAA====", "data:image/png;base64,AA=A", "data:image/png;base64,AA!A", + "data:image/png;base64,iVBORw0KGgo=\n", "data:image/png;base64,iVBORw0KGgp=", + "data:image/png;base64,AB==", "data:image/png;base64,/9j/"}) { + check(!parse_image_data_url(bad, image, error), "invalid data URL accepted"); + check(image.bytes.empty() && image.mime_type.empty(), "failed parse retained image"); + check(!error.empty() && error.find(bad) == std::string::npos, "parse error leaks URL"); + } + check(!parse_image_data_url(png, image, error, 7), "image byte cap ignored"); + check(parse_image_data_url(png, image, error, 8), "exact byte cap rejected"); + + json messages = json::array({{{"role", "user"}, {"content", json::array({ + text_part("before"), image_part(jpeg), text_part("between"), image_part(png), text_part("after")})}}}); + json normalized; + std::vector images; + check(extract_chat_images(messages, IMAGE_PLACEHOLDER, normalized, images, error), "ordered images rejected"); + check(images.size() == 2 && images[0].mime_type == "image/jpeg" && images[1].mime_type == "image/png", "image order differs"); + std::string text; + for (const auto & part : normalized[0]["content"]) text += part.at("text").get(); + check(text == "before" + std::string(IMAGE_PLACEHOLDER) + "between" + IMAGE_PLACEHOLDER + "after", "text/image placement differs"); + check(messages[0]["content"][1]["image_url"]["url"] == jpeg, "input JSON mutated"); + check(normalized.dump().find("base64") == std::string::npos, "normalized messages retain bytes"); + + const json plain = json::array({{{"role", "user"}, {"content", "hello"}}, + {{"role", "assistant"}, {"content", "hi"}, {"tool_calls", json::array()}}}); + check(extract_chat_images(plain, IMAGE_PLACEHOLDER, normalized, images, error) && normalized == plain && images.empty(), "text-only request changed"); + + for (const auto & bad : std::vector{ + json::array({{{"role", "user"}, {"content", IMAGE_PLACEHOLDER}}}), + json::array({{{"role", "user"}, {"content", json::array({text_part("")})}}}), + json::array({{{"role", "assistant"}, {"reasoning_content", IMAGE_PLACEHOLDER}, {"content", "hi"}}}), + json::array({{{"type", "function_call_output"}, {"output", IMAGE_PLACEHOLDER}}}), + json::array({{{"type", "function_call"}, {"arguments", IMAGE_PLACEHOLDER}}}), + json::array({{{"role", "assistant"}, {"tool_calls", json::array({{{"function", {{"arguments", IMAGE_PLACEHOLDER}}}}})}}}), + json::array({{{"role", "assistant"}, {"content", json::array({image_part(png)})}}}), + json::array({{{"role", "user"}, {"content", json::array({{{"type", "image_url"}, {"image_url", 42}}})}}}), + json::array({{{"role", "user"}, {"content", json::array({{{"type", "image_url"}, {"image_url", {{"url", 42}}}}})}}}), + json::array({{{"role", "user"}, {"content", json::array({{{"type", "image_url"}}})}}})}) { + check(!extract_chat_images(bad, IMAGE_PLACEHOLDER, normalized, images, error), "invalid image message accepted"); + check(images.empty() && normalized.is_null(), "failed extraction retains partial state"); + } + ImageInputLimits limits; + limits.image_count = 1; + check(!extract_chat_images(messages, IMAGE_PLACEHOLDER, normalized, images, error, limits), "image count cap ignored"); + limits.image_count = 4; + limits.request_bytes = 10; + check(!extract_chat_images(messages, IMAGE_PLACEHOLDER, normalized, images, error, limits), "aggregate byte cap ignored"); + limits.request_bytes = 11; + check(extract_chat_images(messages, IMAGE_PLACEHOLDER, normalized, images, error, limits), "exact aggregate cap rejected"); + + json deep = plain; + json * nested = &deep[0]["metadata"]; + for (int i = 0; i < 72; ++i) { + *nested = json::object(); + nested = &(*nested)["nested"]; + } + *nested = image_part(png); + check(!extract_chat_images(deep, IMAGE_PLACEHOLDER, normalized, images, error), "deep metadata accepted before copy"); + check(error == "image message nesting exceeds 64 levels", "unexpected depth rejection"); + redact_image_urls(deep); + check((*nested)["image_url"] == "[image omitted]", "deep redaction failed"); + + json status = {{"messages", messages}, {"raw_body", {{"nested", image_part(png)}}}}; + redact_image_urls(status); + check(status.dump().find("base64") == std::string::npos, "status leaks image data URL"); + check(status["messages"][0]["content"][0]["text"] == "before", "redaction alters ordinary text"); + std::cout << "PASS: strict bounded data URLs, ordered content, placeholder rejection, transactional failures, redaction\n"; + } catch (const std::exception & e) { + std::cerr << "FAIL: " << e.what() << '\n'; + return 1; + } +} diff --git a/server/test/test_image_resize.cpp b/server/test/test_image_resize.cpp new file mode 100644 index 000000000..dea4bf06b --- /dev/null +++ b/server/test/test_image_resize.cpp @@ -0,0 +1,276 @@ +// resize_rgb_bicubic must reproduce Pillow's Image.resize(BICUBIC) byte for byte. +// Expected data generated with Pillow 12.2.0. +#include "common/vision/image_resize.h" + +#include +#include +#include + +using luce::vision::resize_rgb_bicubic; + +static int failures = 0; +static void check(bool condition, const char * message) { + if (!condition) { std::fprintf(stderr, "FAIL: %s\n", message); ++failures; } +} + +// 23x17 RGB source with hard edges and noise. +const std::vector SOURCE = { + 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 214, 35, 123, 46, 217, 30, 63, 114, 31, 203, 25, 113, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 32, 30, 105, 254, 218, 160, 238, 232, 185, 153, 127, 92, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 77, 250, 215, 20, 39, 160, 174, 179, 254, 255, 255, 255, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 11, 236, 181, 86, 59, 252, 30, 111, 147, 66, 126, 203, 255, 255, 255, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 194, 118, 77, 42, 90, 77, 118, 119, 6, 248, 93, 134, 255, 255, 255, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 201, 53, 246, 205, 31, 97, 34, 106, 225, 0, 0, 0, 255, 255, 255, + 0, 0, 0, 255, 255, 255, 192, 76, 129, 177, 186, 242, 62, 59, 249, 238, 245, 247, 0, 0, 0, 255, 255, 255, + 0, 0, 0, 255, 255, 255, 13, 152, 46, 133, 187, 85, 182, 114, 168, 114, 99, 122, 0, 0, 0, 255, 255, 255, + 0, 0, 0, 255, 255, 255, 228, 178, 186, 41, 112, 52, 116, 240, 100, 172, 104, 247, 0, 245, 176, 43, 61, 198, + 102, 244, 91, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 238, 74, 242, 179, 79, 67, 10, 7, 52, + 71, 222, 99, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 234, 215, 66, 77, 9, 225, 93, 2, 76, + 88, 72, 242, 255, 255, 255, 0, 0, 0, 255, 255, 255, 21, 50, 231, 14, 32, 226, 166, 102, 141, 231, 244, 126, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 37, 108, 155, 62, 79, 187, 73, 129, 70, 239, 112, 48, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 47, 187, 9, 173, 234, 225, 9, 196, 169, 151, 32, 57, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 216, 132, 207, 76, 253, 167, 45, 142, 29, 93, 217, 37, 255, 255, 255, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 213, 137, 66, 22, 122, 56, 82, 134, 25, 92, 103, 159, 255, 255, 255, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 97, 243, 125, 228, 54, 221, 253, 201, 157, 110, 117, 175, 255, 255, 255, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 137, 228, 1, 134, 186, 168, + 165, 125, 17, 158, 111, 182, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 45, 73, 204, 21, 201, 11, + 153, 155, 119, 43, 79, 199, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 21, 68, 184, 53, 192, 231, + 25, 9, 125, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 105, 118, 235, 252, 195, 39, 245, 147, 23, + 101, 39, 75, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 148, 146, 237, 238, 238, 60, 102, 159, 43, + 242, 8, 148, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 67, 143, 57, 186, 118, 254, 248, 201, 12, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 169, 0, 166, 173, 203, 61, 100, 6, 148, 129, 190, 33, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 127, 136, 223, 161, 97, 191, 219, 14, 204, 104, 41, 25, + 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 152, 130, 133, 207, 122, 154, 247, 201, 61, 85, 82, 38, + 106, 254, 112, 231, 170, 230, 218, 71, 98, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 196, 211, 107, + 192, 138, 173, 31, 255, 142, 184, 64, 110, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 242, 250, 0, + 37, 200, 239, 229, 127, 55, 114, 79, 77, 255, 255, 255, 0, 0, 0, 255, 255, 255, 65, 128, 223, 57, 50, 36, + 153, 98, 198, 133, 114, 0, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 28, 11, 99, 255, 215, 41, + 131, 116, 217, 189, 116, 252, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 159, 99, 118, 238, 113, 135, + 151, 55, 253, 95, 114, 248, 0, 0, 0, 255, 255, 255, 0, 0, 0, 26, 30, 94, 201, 230, 160, 57, 40, 84, + 168, 97, 94, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 41, 179, 215, 63, 106, 194, 182, 158, 221, + 44, 25, 242, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 192, 17, 237, 32, 31, 131, 99, 32, 173, + 185, 139, 171, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, + 77, 120, 167, 163, 235, 185, 40, 101, 200, 81, 126, 208, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, + 215, 255, 228, 88, 119, 68, 213, 235, 120, 62, 150, 150, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, + 144, 96, 167, 33, 202, 128, 125, 118, 51, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 25, 97, 99, + 38, 190, 91, 229, 133, 3, 54, 179, 111, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 190, 94, 159, + 39, 104, 16, 253, 247, 32, 208, 51, 202, 255, 255, 255, 0, 0, 0, 255, 255, 255, 0, 0, 0, 212, 213, 9, + 186, 100, 200, 207, 104, 3, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 72, 203, 45, 189, 87, 74, + 178, 145, 82, 87, 34, 55, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 113, 207, 100, 242, 93, 111, + 21, 204, 80, 196, 183, 63, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, 127, 217, 199, 188, 228, 224, + 91, 11, 1, 250, 238, 120, 228, 234, 91, 242, 204, 54, 34, 65, 183, 255, 255, 255, 0, 0, 0, 255, 255, 255, + 0, 0, 0, 69, 13, 33, 56, 99, 67, 251, 147, 84, 113, 33, 179, 255, 255, 255, 0, 0, 0, 255, 255, 255, + 0, 0, 0, 190, 18, 101, 93, 206, 82, 142, 167, 192, 86, 135, 58, 255, 255, 255, 0, 0, 0, 255, 255, 255, + 188, 74, 184, 169, 41, 226, 117, 90, 24, 151, 129, 158, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, + 116, 23, 11, 27, 1, 181, 155, 54, 182, 114, 211, 154, 0, 0, 0, 255, 255, 255, 0, 0, 0, 255, 255, 255, + 74, 138, 205, 135, 5, 28, 179, 227, 252, 127, 84, 0, 0, 0, 0, 255, 255, 255, 0, 0, 0, +}; + +// Pillow: 9x11 (shrinks both ways, antialiased). +const std::vector PILLOW_9X11 = { + 126, 124, 123, 125, 133, 137, 80, 110, 106, 129, 112, 128, 125, 117, 124, 174, 151, 113, 143, 122, 115, 129, 139, 150, + 110, 107, 200, 118, 130, 127, 134, 144, 158, 115, 129, 220, 135, 124, 136, 114, 145, 112, 125, 127, 94, 130, 102, 116, + 152, 126, 156, 132, 139, 132, 61, 112, 206, 139, 165, 134, 132, 126, 135, 118, 81, 139, 112, 131, 90, 132, 133, 128, + 122, 143, 115, 102, 88, 154, 133, 135, 126, 98, 141, 152, 136, 183, 100, 129, 125, 123, 89, 112, 83, 123, 132, 120, + 113, 129, 125, 119, 150, 131, 163, 148, 166, 114, 126, 145, 128, 127, 125, 132, 156, 121, 178, 141, 77, 127, 113, 122, + 118, 132, 150, 123, 153, 101, 132, 101, 137, 112, 128, 133, 103, 126, 157, 123, 135, 117, 172, 123, 161, 143, 110, 86, + 116, 128, 102, 148, 154, 169, 169, 90, 149, 120, 103, 88, 144, 143, 133, 202, 158, 125, 106, 128, 125, 159, 115, 111, + 122, 124, 125, 148, 149, 121, 144, 130, 164, 126, 123, 124, 168, 150, 123, 140, 105, 162, 124, 125, 127, 100, 103, 116, + 149, 134, 134, 107, 128, 146, 79, 114, 154, 147, 135, 212, 131, 144, 126, 106, 53, 141, 145, 111, 184, 128, 148, 136, + 129, 125, 130, 94, 130, 124, 104, 154, 118, 120, 133, 134, 137, 133, 132, 160, 166, 92, 139, 131, 142, 135, 135, 117, + 152, 138, 95, 155, 153, 109, 134, 160, 129, 133, 108, 72, 95, 86, 82, 162, 148, 147, 138, 148, 96, 121, 109, 90, + 142, 172, 154, 158, 153, 142, 195, 128, 137, 121, 112, 118, 128, 127, 127, 100, 53, 98, 128, 111, 144, 125, 128, 131, + 131, 114, 137, 127, 133, 105, 119, 119, 116, +}; + +// Pillow: 40x31 (enlarges both ways). +const std::vector PILLOW_40X31 = { + 0, 0, 0, 81, 81, 81, 255, 255, 255, 122, 122, 122, 0, 0, 0, 179, 195, 187, 255, 243, 255, 253, 45, 151, + 159, 81, 68, 46, 228, 14, 34, 185, 6, 79, 102, 29, 192, 43, 92, 167, 0, 77, 0, 0, 0, 93, 108, 101, + 255, 255, 255, 100, 100, 100, 0, 0, 0, 214, 214, 207, 228, 230, 254, 33, 39, 122, 109, 91, 118, 255, 230, 168, + 255, 254, 198, 237, 231, 190, 179, 165, 121, 84, 70, 41, 0, 0, 0, 120, 121, 123, 255, 255, 255, 78, 78, 78, + 5, 2, 3, 227, 213, 218, 237, 255, 255, 82, 255, 219, 13, 147, 180, 10, 39, 167, 127, 132, 227, 200, 196, 255, + 76, 76, 76, 111, 111, 111, 178, 178, 178, 126, 126, 126, 78, 78, 78, 146, 151, 147, 179, 168, 180, 166, 102, 150, + 116, 117, 120, 55, 171, 91, 40, 149, 64, 61, 107, 63, 138, 62, 118, 148, 46, 125, 81, 75, 75, 115, 124, 117, + 179, 179, 179, 118, 118, 118, 81, 81, 81, 159, 160, 157, 160, 152, 166, 85, 59, 105, 122, 97, 109, 196, 182, 139, + 203, 206, 139, 203, 194, 129, 195, 140, 111, 145, 97, 91, 76, 75, 76, 121, 126, 127, 179, 179, 179, 110, 110, 110, + 84, 84, 83, 162, 156, 154, 167, 179, 197, 115, 196, 226, 84, 110, 183, 75, 35, 147, 114, 112, 215, 139, 165, 254, + 243, 243, 243, 163, 163, 163, 14, 14, 14, 132, 132, 132, 239, 239, 239, 90, 74, 78, 12, 39, 26, 13, 201, 149, + 44, 182, 215, 77, 70, 232, 53, 84, 176, 31, 115, 131, 45, 103, 169, 117, 145, 214, 237, 243, 243, 153, 151, 145, + 12, 12, 12, 149, 149, 149, 232, 233, 232, 63, 68, 70, 41, 19, 11, 175, 97, 73, 141, 109, 91, 58, 97, 84, + 83, 119, 34, 144, 126, 24, 221, 94, 96, 249, 142, 178, 243, 242, 243, 125, 135, 133, 12, 12, 12, 166, 166, 166, + 223, 224, 222, 48, 57, 44, 48, 12, 70, 177, 63, 237, 212, 46, 185, 190, 30, 106, 91, 81, 189, 32, 113, 243, + 162, 166, 158, 140, 138, 134, 98, 85, 89, 135, 128, 126, 167, 166, 160, 120, 106, 111, 90, 98, 97, 64, 168, 141, + 93, 161, 209, 134, 110, 255, 74, 81, 236, 34, 87, 196, 103, 163, 226, 152, 197, 227, 162, 175, 166, 129, 133, 124, + 88, 93, 93, 137, 140, 138, 167, 166, 165, 113, 105, 116, 96, 91, 75, 120, 128, 54, 98, 136, 62, 71, 129, 79, + 100, 112, 59, 144, 99, 62, 195, 94, 118, 193, 117, 155, 160, 160, 168, 122, 132, 128, 93, 97, 86, 142, 146, 140, + 158, 165, 159, 99, 110, 93, 119, 89, 125, 214, 86, 219, 202, 71, 155, 149, 71, 77, 80, 127, 133, 43, 161, 174, + 2, 3, 1, 89, 88, 87, 251, 248, 249, 125, 123, 122, 9, 9, 7, 173, 180, 177, 252, 232, 244, 201, 98, 142, + 185, 110, 161, 180, 187, 243, 100, 109, 253, 68, 66, 249, 203, 208, 255, 194, 201, 190, 16, 20, 15, 95, 96, 94, + 252, 253, 253, 105, 106, 106, 17, 16, 17, 203, 191, 200, 209, 243, 218, 31, 160, 63, 44, 164, 43, 134, 188, 87, + 172, 144, 139, 176, 107, 165, 141, 108, 143, 73, 64, 80, 1, 1, 4, 122, 124, 122, 252, 253, 250, 87, 88, 86, + 23, 25, 24, 202, 206, 203, 253, 250, 251, 236, 181, 196, 133, 131, 115, 42, 115, 52, 85, 196, 80, 118, 247, 104, + 92, 36, 118, 105, 125, 153, 138, 255, 225, 57, 141, 158, 4, 18, 91, 117, 186, 148, 208, 255, 196, 255, 188, 212, + 188, 131, 162, 76, 81, 98, 127, 131, 200, 180, 174, 255, 130, 136, 150, 103, 82, 94, 112, 21, 111, 181, 86, 144, + 239, 179, 165, 87, 54, 66, 0, 0, 20, 115, 189, 146, 175, 255, 204, 152, 230, 180, 94, 154, 95, 53, 79, 25, + 178, 160, 163, 225, 189, 229, 74, 67, 91, 37, 36, 18, 111, 93, 14, 157, 125, 140, 170, 125, 254, 78, 21, 95, + 49, 0, 40, 144, 131, 224, 215, 220, 255, 255, 246, 241, 123, 135, 110, 0, 46, 7, 133, 194, 124, 214, 255, 191, + 185, 94, 255, 114, 152, 228, 0, 236, 178, 8, 144, 192, 53, 67, 195, 79, 183, 111, 140, 253, 116, 242, 250, 239, + 158, 151, 160, 6, 6, 4, 142, 142, 138, 241, 242, 236, 61, 74, 59, 38, 4, 43, 211, 66, 225, 236, 85, 194, + 173, 74, 68, 73, 24, 47, 9, 27, 56, 40, 163, 71, 128, 246, 141, 248, 247, 249, 147, 134, 146, 6, 5, 8, + 158, 159, 159, 230, 232, 232, 49, 50, 61, 56, 55, 6, 222, 212, 53, 176, 139, 149, 74, 7, 224, 79, 6, 129, + 90, 15, 94, 77, 42, 201, 143, 133, 246, 245, 247, 246, 127, 128, 118, 12, 10, 12, 166, 163, 166, 249, 245, 249, + 75, 65, 252, 40, 76, 238, 0, 92, 211, 57, 81, 195, 141, 96, 174, 184, 210, 134, 168, 220, 100, 86, 95, 77, + 107, 107, 114, 173, 173, 170, 123, 123, 116, 85, 85, 76, 148, 149, 138, 163, 162, 186, 116, 108, 205, 104, 80, 187, + 104, 68, 145, 63, 72, 99, 60, 93, 63, 154, 141, 51, 164, 135, 55, 90, 78, 82, 122, 118, 130, 176, 173, 178, + 113, 114, 114, 83, 85, 85, 156, 152, 163, 169, 193, 138, 125, 203, 27, 126, 184, 110, 125, 153, 226, 56, 145, 181, + 34, 110, 132, 106, 46, 120, 113, 35, 92, 74, 77, 76, 128, 134, 130, 179, 175, 178, 114, 109, 115, 79, 74, 80, + 39, 58, 230, 27, 52, 231, 17, 44, 224, 87, 70, 172, 169, 121, 122, 222, 220, 122, 181, 203, 92, 32, 34, 18, + 92, 89, 101, 238, 237, 237, 115, 115, 115, 24, 23, 25, 184, 180, 177, 213, 228, 232, 73, 134, 153, 33, 91, 153, + 48, 83, 176, 55, 116, 116, 96, 138, 60, 209, 120, 58, 182, 73, 40, 27, 17, 15, 109, 117, 119, 240, 238, 240, + 99, 98, 99, 31, 30, 32, 199, 188, 200, 198, 243, 188, 52, 197, 29, 103, 206, 102, 184, 224, 227, 87, 228, 210, + 44, 185, 156, 133, 72, 83, 109, 11, 30, 17, 17, 16, 130, 136, 133, 237, 235, 235, 94, 94, 93, 18, 18, 16, + 188, 112, 230, 137, 144, 217, 52, 190, 182, 55, 166, 113, 85, 146, 52, 103, 198, 32, 139, 224, 76, 191, 191, 189, + 145, 131, 156, 61, 57, 59, 133, 133, 138, 186, 190, 197, 90, 97, 99, 80, 71, 58, 173, 132, 86, 118, 129, 89, + 12, 102, 77, 44, 129, 48, 94, 142, 44, 109, 98, 96, 147, 117, 150, 200, 195, 187, 144, 138, 136, 69, 56, 70, + 146, 141, 150, 184, 183, 184, 96, 83, 86, 59, 110, 55, 78, 238, 94, 146, 180, 162, 218, 91, 220, 220, 176, 199, + 193, 207, 165, 124, 106, 138, 137, 114, 154, 201, 198, 189, 134, 128, 119, 68, 58, 56, 155, 152, 146, 201, 203, 194, + 187, 95, 158, 165, 163, 172, 129, 255, 191, 61, 192, 93, 22, 109, 6, 79, 183, 44, 160, 248, 117, 248, 255, 208, + 160, 173, 144, 8, 22, 21, 143, 145, 133, 244, 237, 217, 77, 72, 83, 37, 10, 14, 159, 87, 32, 147, 135, 66, + 76, 160, 95, 56, 125, 36, 64, 99, 22, 99, 123, 141, 160, 172, 229, 230, 235, 255, 115, 138, 134, 0, 24, 0, + 153, 170, 145, 236, 233, 234, 45, 47, 75, 0, 37, 25, 61, 180, 88, 158, 153, 173, 246, 90, 234, 230, 125, 158, + 198, 156, 121, 151, 154, 184, 172, 189, 238, 225, 234, 255, 101, 124, 140, 0, 26, 32, 140, 150, 173, 218, 214, 247, + 20, 3, 14, 98, 98, 99, 244, 255, 255, 111, 135, 117, 0, 10, 0, 164, 179, 167, 248, 255, 213, 179, 253, 53, + 130, 210, 60, 103, 157, 147, 144, 148, 97, 178, 143, 60, 158, 116, 139, 101, 65, 124, 18, 2, 0, 107, 108, 90, + 237, 252, 240, 92, 105, 89, 9, 16, 0, 190, 193, 188, 223, 228, 255, 95, 112, 230, 21, 116, 113, 1, 166, 3, + 114, 181, 78, 167, 162, 160, 60, 94, 180, 0, 40, 117, 0, 19, 5, 133, 129, 127, 255, 238, 255, 107, 87, 94, + 43, 35, 27, 215, 213, 211, 208, 220, 255, 68, 101, 210, 31, 126, 198, 33, 161, 190, 43, 79, 165, 46, 28, 148, + 76, 83, 77, 110, 110, 108, 174, 162, 168, 132, 121, 128, 91, 84, 93, 150, 135, 158, 168, 167, 161, 120, 193, 74, + 136, 204, 86, 183, 203, 139, 190, 164, 54, 188, 122, 3, 167, 99, 118, 129, 83, 156, 79, 79, 93, 116, 122, 122, + 180, 172, 177, 125, 119, 127, 91, 88, 93, 159, 156, 145, 153, 154, 183, 73, 85, 218, 68, 145, 131, 96, 234, 22, + 115, 200, 37, 125, 139, 97, 120, 78, 185, 101, 54, 176, 86, 72, 87, 124, 128, 117, 164, 177, 165, 103, 111, 109, + 84, 86, 88, 163, 159, 152, 131, 148, 164, 21, 81, 133, 46, 131, 204, 101, 180, 255, 89, 96, 138, 80, 48, 67, + 248, 249, 248, 165, 165, 165, 9, 8, 9, 132, 131, 132, 244, 244, 245, 82, 79, 73, 6, 10, 34, 78, 104, 200, + 163, 164, 176, 245, 194, 52, 254, 182, 16, 234, 140, 19, 132, 52, 52, 132, 82, 131, 244, 237, 244, 152, 157, 154, + 7, 6, 6, 150, 150, 150, 238, 237, 237, 63, 63, 55, 19, 18, 49, 122, 121, 221, 195, 197, 175, 228, 242, 51, + 145, 210, 27, 108, 139, 53, 202, 26, 125, 248, 73, 198, 249, 241, 249, 126, 140, 131, 7, 7, 7, 167, 168, 168, + 229, 228, 229, 53, 48, 53, 3, 28, 6, 52, 130, 51, 117, 135, 175, 183, 126, 254, 221, 167, 98, 238, 194, 4, + 156, 156, 158, 138, 135, 139, 104, 96, 104, 126, 123, 125, 146, 151, 146, 112, 125, 110, 97, 88, 124, 115, 42, 210, + 170, 107, 160, 234, 209, 47, 207, 142, 51, 171, 69, 72, 129, 93, 50, 121, 124, 73, 148, 147, 151, 128, 130, 137, + 98, 101, 99, 134, 127, 130, 156, 144, 149, 112, 114, 107, 101, 110, 128, 130, 134, 224, 183, 166, 200, 228, 183, 131, + 180, 129, 112, 150, 68, 102, 191, 22, 89, 185, 51, 112, 147, 142, 160, 123, 129, 132, 107, 98, 96, 134, 134, 136, + 143, 149, 154, 112, 113, 118, 97, 111, 92, 98, 134, 71, 154, 125, 156, 221, 125, 216, 253, 182, 84, 255, 219, 5, + 0, 0, 0, 88, 88, 88, 252, 252, 252, 123, 123, 123, 5, 6, 5, 175, 187, 175, 255, 220, 255, 184, 27, 189, + 168, 64, 118, 174, 202, 62, 131, 95, 110, 100, 12, 144, 128, 147, 70, 97, 156, 11, 1, 10, 0, 103, 99, 110, + 255, 255, 255, 104, 104, 104, 12, 12, 11, 195, 194, 188, 241, 246, 255, 139, 151, 230, 129, 116, 207, 162, 96, 191, + 206, 43, 210, 210, 13, 184, 138, 36, 66, 59, 23, 0, 0, 0, 0, 123, 127, 129, 254, 254, 254, 85, 85, 85, + 21, 22, 22, 208, 210, 209, 241, 237, 236, 157, 138, 140, 170, 113, 141, 210, 125, 151, 237, 176, 90, 250, 207, 54, + 33, 19, 0, 85, 114, 59, 185, 255, 189, 155, 192, 162, 126, 83, 118, 213, 133, 161, 255, 169, 203, 247, 161, 237, + 164, 120, 144, 56, 73, 12, 132, 119, 135, 190, 152, 225, 83, 96, 63, 41, 76, 0, 91, 107, 33, 171, 156, 131, + 230, 199, 232, 85, 167, 135, 7, 136, 62, 167, 147, 140, 247, 181, 206, 222, 224, 255, 133, 131, 176, 50, 17, 74, + 188, 105, 197, 246, 147, 238, 71, 30, 51, 32, 24, 0, 120, 124, 0, 138, 204, 124, 134, 243, 255, 120, 125, 101, + 128, 65, 8, 175, 141, 119, 215, 198, 195, 235, 221, 227, 147, 119, 132, 78, 40, 46, 203, 176, 138, 255, 251, 187, + 85, 75, 47, 85, 138, 61, 93, 237, 96, 171, 222, 184, 240, 164, 234, 222, 82, 125, 221, 99, 111, 241, 236, 233, + 149, 164, 161, 11, 10, 15, 141, 142, 141, 236, 240, 235, 70, 67, 78, 34, 36, 18, 170, 185, 93, 214, 188, 137, + 195, 139, 161, 94, 217, 161, 44, 242, 147, 138, 98, 112, 214, 97, 149, 243, 236, 239, 137, 146, 139, 10, 13, 9, + 155, 161, 155, 224, 231, 228, 51, 55, 71, 66, 64, 15, 237, 234, 8, 158, 244, 118, 43, 191, 232, 164, 153, 138, + 226, 113, 58, 125, 71, 64, 148, 130, 139, 242, 243, 243, 124, 128, 126, 11, 15, 14, 159, 161, 165, 238, 240, 246, + 79, 124, 184, 66, 116, 128, 53, 103, 46, 129, 119, 144, 198, 131, 221, 179, 108, 84, 134, 85, 25, 68, 69, 63, + 108, 117, 120, 184, 183, 189, 121, 122, 121, 74, 77, 71, 158, 158, 160, 169, 164, 180, 93, 79, 111, 151, 112, 73, + 246, 193, 68, 162, 190, 142, 99, 160, 206, 176, 117, 205, 168, 82, 165, 72, 66, 72, 115, 122, 112, 185, 188, 184, + 109, 115, 109, 72, 80, 75, 157, 164, 169, 199, 191, 174, 194, 167, 81, 187, 157, 106, 180, 143, 170, 193, 105, 192, + 176, 83, 194, 109, 92, 192, 71, 82, 149, 67, 68, 71, 130, 132, 124, 182, 185, 184, 105, 107, 111, 63, 65, 71, + 57, 121, 231, 61, 99, 160, 72, 65, 51, 106, 70, 117, 138, 88, 176, 142, 111, 62, 106, 92, 9, 26, 23, 22, + 100, 101, 109, 231, 230, 230, 116, 115, 115, 32, 31, 28, 185, 184, 176, 198, 197, 223, 35, 42, 137, 90, 84, 76, + 227, 201, 53, 192, 163, 147, 146, 111, 239, 169, 110, 255, 126, 76, 198, 25, 20, 31, 112, 116, 106, 233, 232, 233, + 99, 99, 101, 35, 36, 37, 187, 191, 187, 227, 207, 223, 165, 89, 154, 186, 76, 124, 217, 91, 129, 173, 59, 210, + 131, 56, 255, 116, 110, 254, 71, 87, 170, 23, 23, 26, 132, 131, 122, 231, 228, 230, 96, 95, 96, 23, 23, 24, + 24, 42, 132, 79, 100, 130, 163, 183, 126, 124, 123, 122, 81, 50, 109, 123, 66, 65, 172, 118, 91, 206, 198, 194, + 147, 147, 149, 49, 44, 48, 138, 133, 131, 206, 200, 193, 104, 92, 80, 47, 62, 71, 34, 139, 188, 53, 145, 193, + 90, 120, 154, 159, 148, 199, 178, 145, 243, 84, 49, 248, 89, 66, 240, 191, 186, 198, 138, 138, 126, 54, 52, 56, + 145, 143, 150, 188, 188, 193, 80, 86, 74, 88, 47, 91, 190, 33, 222, 143, 29, 188, 64, 30, 120, 87, 29, 172, + 124, 49, 207, 150, 104, 187, 182, 162, 194, 202, 206, 201, 126, 124, 119, 60, 48, 53, 154, 150, 155, 203, 205, 210, + 0, 0, 48, 86, 94, 104, 229, 253, 199, 130, 142, 124, 30, 16, 47, 138, 89, 104, 222, 173, 179, 238, 241, 253, + 151, 167, 170, 26, 42, 30, 132, 150, 162, 207, 226, 255, 65, 69, 97, 0, 22, 45, 16, 121, 141, 56, 152, 201, + 103, 143, 228, 128, 128, 185, 129, 108, 164, 91, 75, 227, 136, 130, 255, 255, 255, 255, 153, 166, 144, 5, 11, 0, + 168, 172, 151, 244, 255, 228, 47, 83, 61, 14, 6, 45, 132, 0, 167, 123, 42, 188, 79, 88, 168, 55, 34, 129, + 83, 32, 124, 185, 141, 174, 246, 212, 227, 248, 233, 255, 105, 126, 128, 0, 37, 12, 151, 170, 150, 241, 238, 223, + 0, 0, 0, 84, 85, 86, 255, 255, 255, 124, 125, 123, 0, 0, 0, 180, 170, 169, 247, 246, 255, 125, 154, 198, + 113, 160, 170, 149, 208, 165, 100, 165, 197, 54, 116, 221, 70, 114, 211, 49, 77, 140, 0, 5, 17, 97, 109, 111, + 245, 251, 255, 105, 105, 114, 17, 14, 22, 180, 175, 200, 248, 249, 255, 235, 255, 250, 153, 192, 157, 78, 103, 59, + 168, 186, 104, 207, 236, 147, 87, 172, 141, 13, 80, 87, 7, 0, 12, 125, 107, 130, 240, 242, 254, 76, 73, 87, + 19, 13, 26, 214, 210, 210, 255, 229, 255, 167, 117, 196, 75, 138, 146, 19, 182, 103, 95, 154, 85, 145, 131, 75, + 103, 103, 97, 120, 119, 118, 155, 153, 158, 126, 125, 127, 101, 102, 99, 147, 147, 143, 144, 152, 162, 55, 98, 141, + 65, 150, 148, 121, 240, 165, 113, 169, 138, 92, 100, 125, 78, 145, 178, 78, 148, 172, 97, 92, 97, 132, 117, 111, + 165, 162, 156, 120, 120, 116, 93, 95, 90, 143, 146, 131, 177, 166, 162, 210, 188, 208, 142, 158, 141, 67, 123, 49, + 170, 199, 51, 229, 232, 92, 151, 150, 170, 100, 102, 165, 90, 100, 87, 127, 134, 117, 167, 167, 160, 121, 124, 115, + 100, 106, 97, 143, 149, 148, 166, 153, 151, 167, 127, 97, 126, 151, 132, 91, 173, 159, 129, 130, 65, 157, 103, 14, + 252, 252, 252, 166, 166, 166, 6, 6, 6, 132, 132, 132, 247, 247, 247, 86, 80, 80, 3, 2, 4, 18, 72, 86, + 24, 140, 111, 35, 194, 97, 152, 162, 40, 217, 131, 11, 91, 160, 74, 92, 206, 160, 241, 252, 251, 157, 147, 152, + 3, 3, 3, 150, 150, 150, 239, 240, 240, 57, 64, 59, 35, 7, 27, 181, 86, 153, 128, 99, 111, 35, 106, 14, + 167, 206, 7, 255, 231, 54, 217, 84, 165, 221, 100, 239, 252, 247, 252, 128, 138, 128, 4, 4, 4, 169, 169, 169, + 230, 230, 233, 41, 41, 55, 48, 51, 2, 200, 200, 2, 208, 169, 112, 180, 98, 195, 198, 100, 71, 209, 107, 1, + 134, 135, 142, 125, 125, 133, 108, 107, 116, 121, 122, 132, 135, 137, 145, 131, 122, 126, 108, 116, 97, 40, 139, 62, + 51, 145, 64, 110, 143, 82, 182, 139, 44, 205, 132, 19, 109, 118, 57, 80, 120, 100, 139, 149, 146, 139, 137, 139, + 118, 115, 117, 127, 129, 133, 132, 139, 142, 118, 124, 119, 120, 118, 112, 140, 136, 119, 148, 116, 97, 147, 104, 64, + 130, 182, 39, 138, 218, 49, 209, 143, 119, 208, 119, 151, 140, 147, 143, 120, 130, 130, 116, 109, 116, 132, 128, 130, + 138, 135, 137, 118, 111, 124, 128, 142, 115, 163, 217, 83, 194, 210, 169, 207, 162, 224, 167, 80, 65, 143, 38, 0, + 1, 0, 0, 90, 90, 87, 255, 255, 253, 125, 125, 122, 5, 5, 2, 182, 172, 185, 236, 255, 235, 98, 224, 79, + 103, 155, 41, 186, 85, 71, 196, 119, 84, 172, 142, 85, 118, 68, 67, 53, 9, 32, 0, 0, 0, 104, 108, 107, + 255, 255, 255, 105, 104, 103, 13, 10, 10, 197, 188, 198, 237, 255, 238, 124, 222, 118, 170, 151, 97, 240, 91, 110, + 101, 155, 97, 31, 210, 80, 163, 202, 68, 144, 121, 32, 3, 0, 0, 118, 118, 125, 255, 255, 255, 84, 86, 85, + 21, 19, 20, 210, 205, 205, 234, 255, 255, 134, 221, 206, 153, 232, 221, 183, 218, 212, 123, 79, 73, 84, 3, 5, + 143, 141, 56, 181, 186, 92, 254, 255, 160, 205, 197, 82, 136, 114, 27, 121, 126, 162, 142, 173, 237, 200, 255, 201, + 150, 167, 111, 65, 13, 11, 168, 133, 122, 231, 221, 201, 84, 61, 64, 3, 0, 0, 17, 0, 1, 75, 72, 69, + 150, 181, 147, 152, 131, 74, 148, 79, 43, 171, 99, 175, 196, 161, 239, 216, 255, 216, 158, 155, 123, 89, 19, 31, + 138, 161, 143, 159, 243, 191, 81, 113, 46, 63, 33, 0, 103, 0, 34, 145, 114, 105, 166, 254, 169, 101, 148, 125, + 76, 87, 100, 140, 170, 134, 193, 229, 193, 221, 255, 255, 135, 170, 171, 59, 78, 71, 151, 136, 134, 198, 166, 171, + 255, 234, 133, 245, 229, 123, 221, 216, 98, 243, 217, 64, 230, 195, 58, 83, 92, 133, 64, 90, 203, 222, 221, 241, + 163, 159, 157, 16, 21, 21, 137, 140, 142, 228, 230, 233, 82, 88, 88, 20, 20, 19, 69, 9, 26, 57, 38, 46, + 43, 81, 68, 174, 130, 82, 250, 137, 105, 139, 53, 151, 134, 85, 201, 236, 225, 238, 139, 145, 138, 12, 19, 19, + 158, 155, 158, 227, 220, 226, 57, 66, 67, 48, 17, 31, 179, 21, 109, 151, 102, 93, 88, 187, 71, 130, 193, 167, + 144, 171, 190, 86, 128, 64, 131, 161, 99, 237, 233, 232, 126, 120, 125, 17, 16, 16, 162, 164, 164, 239, 243, 243, + 214, 127, 169, 205, 113, 182, 185, 92, 189, 172, 109, 100, 154, 127, 36, 128, 117, 124, 100, 95, 152, 72, 67, 79, + 115, 113, 113, 181, 184, 185, 119, 121, 123, 74, 75, 77, 155, 163, 164, 178, 164, 158, 121, 41, 31, 62, 10, 55, + 28, 21, 144, 122, 53, 160, 190, 94, 160, 136, 147, 165, 91, 137, 134, 71, 72, 73, 120, 118, 120, 180, 185, 184, + 114, 112, 114, 82, 76, 80, 160, 160, 159, 170, 177, 189, 116, 115, 188, 107, 70, 104, 120, 59, 39, 159, 165, 178, + 169, 210, 222, 127, 122, 54, 90, 72, 26, 71, 69, 69, 129, 128, 133, 180, 179, 179, 110, 112, 112, 73, 76, 76, + 184, 64, 186, 179, 48, 213, 166, 25, 241, 130, 48, 120, 108, 86, 22, 156, 132, 121, 123, 102, 125, 0, 0, 0, + 87, 88, 87, 255, 255, 255, 111, 111, 111, 0, 0, 0, 196, 204, 205, 255, 242, 234, 147, 58, 34, 65, 0, 61, + 25, 0, 189, 92, 10, 203, 152, 68, 188, 136, 202, 174, 70, 169, 98, 0, 0, 0, 111, 104, 109, 255, 255, 255, + 88, 88, 88, 0, 0, 0, 219, 214, 209, 238, 255, 255, 78, 166, 228, 81, 53, 110, 141, 0, 26, 174, 148, 183, + 179, 227, 235, 152, 121, 51, 69, 26, 0, 0, 0, 0, 131, 135, 140, 255, 255, 255, 81, 81, 81, 0, 0, 0, +}; + +// Pillow: 23x5 (height only). +const std::vector PILLOW_23X5 = { + 104, 96, 126, 126, 165, 164, 92, 91, 117, 153, 175, 147, 143, 136, 159, 94, 128, 162, 74, 113, 162, 143, 123, 167, + 116, 93, 124, 159, 142, 146, 82, 83, 89, 148, 167, 143, 109, 123, 94, 120, 138, 97, 180, 159, 129, 155, 97, 104, + 118, 119, 87, 144, 138, 171, 90, 84, 96, 147, 139, 167, 183, 162, 215, 89, 57, 87, 121, 180, 196, 113, 84, 179, + 82, 177, 196, 77, 92, 92, 160, 227, 119, 153, 167, 134, 121, 124, 114, 165, 147, 129, 114, 104, 118, 126, 95, 118, + 107, 124, 134, 61, 85, 47, 157, 155, 129, 130, 144, 170, 91, 130, 73, 154, 155, 144, 95, 84, 121, 92, 169, 66, + 180, 134, 215, 102, 125, 117, 143, 111, 162, 131, 141, 156, 91, 119, 131, 139, 139, 135, 88, 94, 93, 140, 175, 133, + 131, 112, 134, 171, 131, 123, 159, 122, 169, 152, 150, 71, 173, 117, 115, 107, 101, 72, 109, 114, 95, 177, 164, 149, + 86, 138, 123, 167, 128, 167, 140, 146, 211, 132, 135, 98, 168, 121, 151, 116, 47, 90, 138, 129, 80, 142, 161, 176, + 138, 97, 107, 143, 139, 154, 124, 143, 141, 136, 110, 157, 198, 173, 105, 60, 76, 108, 152, 173, 136, 100, 80, 120, + 168, 133, 110, 115, 146, 150, 100, 140, 112, 144, 145, 149, 77, 110, 120, 69, 108, 127, 163, 163, 153, 113, 125, 151, + 142, 111, 203, 185, 184, 178, 85, 93, 71, 189, 204, 130, 104, 104, 119, 141, 80, 124, 136, 126, 162, 110, 63, 135, + 157, 146, 183, 172, 163, 145, 93, 126, 119, 160, 137, 102, 164, 131, 118, 190, 160, 162, 140, 124, 57, 122, 130, 176, + 102, 153, 111, 122, 122, 100, 164, 142, 107, 92, 100, 99, 89, 47, 49, 99, 107, 142, 153, 94, 103, 139, 140, 166, + 147, 159, 139, 134, 104, 92, 132, 183, 107, 148, 122, 113, 111, 76, 117, 139, 144, 108, 123, 147, 155, 135, 140, 92, + 152, 175, 139, 140, 145, 157, 140, 100, 77, +}; + +int main() { + std::vector out; + std::string error; + check(resize_rgb_bicubic(SOURCE, 23, 17, 9, 11, out, error) && out == PILLOW_9X11, "shrink matches Pillow"); + check(resize_rgb_bicubic(SOURCE, 23, 17, 40, 31, out, error) && out == PILLOW_40X31, "enlarge matches Pillow"); + check(resize_rgb_bicubic(SOURCE, 23, 17, 23, 5, out, error) && out == PILLOW_23X5, "one-axis resize matches Pillow"); + check(resize_rgb_bicubic(SOURCE, 23, 17, 23, 17, out, error) && out == SOURCE, "same size is a copy"); + check(!resize_rgb_bicubic(SOURCE, 23, 17, 0, 5, out, error) && !error.empty(), "zero output size is rejected"); + check(!resize_rgb_bicubic(SOURCE, 24, 17, 9, 11, out, error), "input size mismatch is rejected"); + if (failures) { std::fprintf(stderr, "%d failure(s)\n", failures); return 1; } + std::printf("OK\n"); + return 0; +} diff --git a/server/test/test_moe_source_page_range.cpp b/server/test/test_moe_source_page_range.cpp new file mode 100644 index 000000000..dde857d4b --- /dev/null +++ b/server/test/test_moe_source_page_range.cpp @@ -0,0 +1,148 @@ +#include "../src/common/moe_source_page_range.h" +#include "../src/common/copied_source_reclaim.h" +#include "../src/common/copied_source_upload.h" + +#include +#include +#include +#include +#include +#include +#if defined(__linux__) +#include +#include +#include +#endif + +using namespace luce::common; + +static void check(bool ok, const char * message) { + if (!ok) { std::fprintf(stderr, "FAIL: %s\n", message); std::exit(1); } +} + +static void check_staged_upload() { + const size_t payload = 2 * COPIED_SOURCE_UPLOAD_CHUNK + 37; + const size_t prefix = 17; + std::vector source(prefix + payload + 19), scratch, destination(payload); + for (size_t i = 0; i < source.size(); ++i) + source[i] = static_cast((i * 37 + i / 4096) % 251); + size_t next = 0, calls = 0; + const uint8_t * scratch_base = nullptr; + auto upload = [&](const uint8_t * bytes, size_t offset, size_t count) { + check(offset == next, "staged destination offsets are contiguous and tensor-relative"); + check(count > 0 && count <= COPIED_SOURCE_UPLOAD_CHUNK && count <= payload - offset, + "staged chunk bounds"); + check(bytes == scratch.data(), "backend receives heap scratch"); + const uintptr_t address = reinterpret_cast(bytes); + const uintptr_t source_address = reinterpret_cast(source.data()); + check(address < source_address || address >= source_address + source.size(), + "backend never receives original source bytes"); + if (!scratch_base) scratch_base = bytes; + check(bytes == scratch_base, "one buffer reused for every chunk"); + std::memcpy(destination.data() + offset, bytes, count); + next += count; + ++calls; + }; + check(upload_copied_file_chunks(source.data(), source.size(), prefix, payload, scratch, upload), + "valid multichunk upload"); + check(next == payload && calls == 3, "two full chunks and short final chunk"); + check(std::memcmp(destination.data(), source.data() + prefix, payload) == 0, + "all uploaded bytes preserved"); + auto unexpected = [&](const uint8_t *, size_t, size_t) { check(false, "invalid/empty upload callback"); }; + check(upload_copied_file_chunks(source.data(), source.size(), source.size(), 0, scratch, unexpected), + "empty end span does not upload"); + check(!upload_copied_file_chunks(source.data(), source.size(), source.size() + 1, 0, scratch, unexpected), + "source offset past mapping rejected"); + check(!upload_copied_file_chunks(source.data(), source.size(), prefix, + std::numeric_limits::max(), scratch, unexpected), "overflowing source size rejected"); + check(!upload_copied_file_chunks(nullptr, 0, 0, 0, scratch, unexpected), "null mapping rejected"); + check(!upload_copied_file_chunks(source.data(), std::numeric_limits::max(), + 0, 1, scratch, unexpected), "mapping address overflow rejected"); + size_t small_calls = 0; + check(upload_copied_file_chunks(source.data(), source.size(), prefix, 29, scratch, + [&](const uint8_t * bytes, size_t offset, size_t count) { + check(offset == 0 && count == 29 && bytes == scratch_base, "scratch reused across tensors"); + check(std::memcmp(bytes, source.data() + prefix, count) == 0, "short tensor bytes preserved"); + ++small_calls; + }), "short next tensor upload"); + check(small_calls == 1, "short tensor uploaded once"); +} + +int main() { + check_staged_upload(); + constexpr size_t page = 4096; + MoeSourcePageRange range; + check(moe_source_page_range(page, 5 * page, page + 17, 3 * page + 100, page, range), "unaligned tensor valid"); + check(range.address == 2 * page && range.size == 2 * page, "only tensor interior pages selected"); + check(moe_source_page_range(page, 2 * page, page, 2 * page, page, range) && range.size == 2 * page, + "exact mapping boundaries"); + check(moe_source_page_range(page, page, page + 1, page - 2, page, range) && range.size == 0, + "partial page never reclaimed"); + check(moe_source_page_range(page, page, 2 * page, 0, page, range) && range.size == 0, "empty end span"); + check(!moe_source_page_range(page, page, page - 1, page, page, range), "span before mapping rejected"); + check(!moe_source_page_range(page, page, page, page + 1, page, range), "span past mapping rejected"); + check(!moe_source_page_range(page, page, page, page, 0, range), "zero page size rejected"); + check(!moe_source_page_range(page, page, page, page, 3, range), "invalid page size rejected"); + const uintptr_t max = std::numeric_limits::max(); + check(!moe_source_page_range(max - page, page + 1, max - page, 1, page, range), "mapping overflow rejected"); + check(!moe_source_page_range(page, page, page, std::numeric_limits::max(), page, range), + "tensor overflow rejected"); + check(moe_source_page_range(max - 3 * page, 2 * page, max - 3 * page + 1, page, page, range), + "valid near-address-limit span"); + for (unsigned mask = 0; mask < 16; ++mask) { + check(moe_source_pageout_eligible(mask & 1, mask & 2, mask & 4, mask & 8) == (mask == 15), + "CPU/unmaterialized/unallocated modes excluded"); + } +#if defined(__linux__) + const long raw_page = ::sysconf(_SC_PAGESIZE); + check(raw_page > 0, "native page size"); + const size_t p = static_cast(raw_page), size = 5 * p; + char name[] = "/tmp/ds4v-source-pageout-XXXXXX"; + const int fd = ::mkstemp(name); + check(fd >= 0, "temporary backing file"); + ::unlink(name); + check(::ftruncate(fd, static_cast(size)) == 0, "size backing file"); + void * writable = ::mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + check(writable != MAP_FAILED, "initialize file mapping"); + std::vector expected(size); + for (size_t i = 0; i < size; ++i) expected[i] = static_cast((i * 37 + i / p) % 251); + for (size_t i = 0; i < size; ++i) static_cast(writable)[i] = expected[i]; + check(::msync(writable, size, MS_SYNC) == 0, "clean backing pages"); + check(::munmap(writable, size) == 0, "close initialization mapping"); + void * mapped = ::mmap(nullptr, size, PROT_READ, MAP_PRIVATE, fd, 0); + check(mapped != MAP_FAILED, "read-only source mapping"); + // Keep the original descriptor until copied-source advice returns. + const volatile uint8_t * bytes = static_cast(mapped); + for (size_t i = 0; i < size; ++i) check(bytes[i] == expected[i], "source before advice"); + check(moe_source_page_range(reinterpret_cast(mapped), size, + reinterpret_cast(mapped) + 17, 3 * p + 100, p, range), "native source range"); + std::vector before(5), after(5); + const int before_rc = ::mincore(mapped, size, before.data()); + errno = EBUSY; // Successful advice must not report a stale errno. + const auto advice = reclaim_copied_file_source(mapped, size, + static_cast(mapped) + 17, 3 * p + 100, fd, "test"); + const int after_rc = ::mincore(mapped, size, after.data()); + unsigned before_count = 0, after_count = 0; + for (size_t i = 0; i < 5; ++i) { before_count += before[i] & 1; after_count += after[i] & 1; } + std::printf("copied source requested=%zu madvise_error=%d fadvise_error=%d mincore_before_rc=%d pages=%u after_rc=%d pages=%u\n", + advice.requested, advice.madvise_error, advice.fadvise_error, + before_rc, before_count, after_rc, after_count); + check(advice.range_error == 0 && advice.requested == 2 * p, "advice uses exact interior range"); + check(advice.madvise_error == 0, "read-only file MADV_DONTNEED accepted"); + check(advice.fadvise_error == 0 || advice.fadvise_error == ENOSYS || advice.fadvise_error == EOPNOTSUPP, + "unexpected file cache advice failure"); + const auto partial = reclaim_copied_file_source(mapped, size, + static_cast(mapped) + 1, p - 2, fd, "partial-test"); + check(partial.requested == 0 && partial.range_error == 0, "empty interior causes no whole-file fadvise"); + const auto invalid = reclaim_copied_file_source(mapped, size, mapped, size, -1, "invalid-fd-test"); + check(invalid.requested == 0 && invalid.range_error == EINVAL, "invalid fd rejected before advice"); + check(::close(fd) == 0, "close borrowed fd after advice"); + // Refault correctness is mandatory; eviction count is only diagnostic. + for (size_t i = 0; i < size; ++i) check(bytes[i] == expected[i], "source/refault and edge bytes preserved"); + check(::munmap(mapped, size) == 0, "close source mapping"); +#else + std::puts("SKIP: Linux copied-source advice unavailable; bounds and mode checks passed"); +#endif + std::puts("PASS: staged-copy bytes/bounds, source-page bounds, mode exclusions and supported file-refault checks"); + return 0; +} diff --git a/server/test/test_qwen35_image.cpp b/server/test/test_qwen35_image.cpp new file mode 100644 index 000000000..75ecc2912 --- /dev/null +++ b/server/test/test_qwen35_image.cpp @@ -0,0 +1,122 @@ +// Qwen3.5 image input, the parts that need no GPU: target size, tower patch +// order, position tables, prompt expansion and rotary positions. +#include "qwen35/qwen35_image_prompt.h" +#include "qwen35/qwen35_vision.h" + +#include +#include +#include + +using namespace luce::common; +using namespace luce::vision; + +static int failures = 0; +static void check(bool condition, const char * message) { + if (!condition) { std::fprintf(stderr, "FAIL: %s\n", message); ++failures; } +} + +static Qwen35VisionConfig config() { + Qwen35VisionConfig c; + c.patch_size = 16; + c.merge = 2; + return c; +} + +// Expected sizes computed with the model's reference smart_resize +// (factor 32, 64 to 1024 image tokens). +static void test_target_size() { + const Qwen35VisionConfig c = config(); + int w = 0, h = 0; + std::string error; + check(qwen35_vision_target_size(c, 640, 480, w, h, error) && w == 640 && h == 480, "aligned size is kept"); + check(qwen35_vision_target_size(c, 4000, 3000, w, h, error) && w == 1152 && h == 864, "large image shrinks under the cap"); + check(qwen35_vision_target_size(c, 100, 50, w, h, error) && w == 384 && h == 192, "small image grows to the floor"); + check(qwen35_vision_target_size(c, 304, 272, w, h, error) && w == 320 && h == 256, "halves round to even multiples"); + check(!qwen35_vision_target_size(c, 3000, 10, w, h, error) && !error.empty(), "extreme aspect ratio is rejected"); + check(!qwen35_vision_target_size(c, 0, 10, w, h, error), "empty image is rejected"); +} + +static void test_tower_order() { + // 4 patches across, 2 down: two 2x2 blocks, each listed row by row. + std::vector rope; + qwen35_vision_rope_positions(4, 2, 2, rope); + const std::vector y = {0, 0, 1, 1, 0, 0, 1, 1}, x = {0, 1, 0, 1, 2, 3, 2, 3}; + bool ok = rope.size() == 32; + for (int i = 0; ok && i < 8; ++i) { + ok = rope[i] == y[i] && rope[8 + i] == x[i] && rope[16 + i] == y[i] && rope[24 + i] == x[i]; + } + check(ok, "rotary positions follow merged patch order"); + + // A 4x4 table read on a 4x4 grid is the table itself, in merged order. + std::vector table(16), out; + for (int i = 0; i < 16; ++i) table[i] = float(i); + qwen35_vision_position_rows(table, 4, 1, 4, 4, 2, out); + const std::vector same = {0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15}; + check(out == same, "matching grid reads the table unchanged"); + + // On a 2x2 grid aligned corners land on the table's corners. + qwen35_vision_position_rows(table, 4, 1, 2, 2, 2, out); + check(out == std::vector({0, 3, 12, 15}), "coarser grid samples aligned corners"); + + // 4 points across a 3-wide table: 0, 2/3, 4/3, 2. + std::vector ramp = {0, 1, 2, 0, 1, 2, 0, 1, 2}; + qwen35_vision_position_rows(ramp, 3, 1, 4, 2, 2, out); + bool close = out.size() == 8; + const float expected[8] = {0.0f, 2.0f / 3, 0.0f, 2.0f / 3, 4.0f / 3, 2.0f, 4.0f / 3, 2.0f}; + for (int i = 0; close && i < 8; ++i) close = out[i] > expected[i] - 1e-5f && out[i] < expected[i] + 1e-5f; + check(close, "finer grid interpolates linearly"); +} + +static void test_prompt() { + constexpr int32_t PAD = 9; + std::vector tokens = {1, 2, PAD, 3, PAD, 4}; + std::vector slots(2); + slots[0].columns = 3; slots[0].rows = 2; + slots[1].columns = 1; slots[1].rows = 2; + std::string error; + check(qwen35_expand_image_tokens(tokens, PAD, slots, 64, error), "expansion succeeds"); + check(tokens == std::vector({1, 2, PAD, PAD, PAD, PAD, PAD, PAD, 3, PAD, PAD, 4}), "one pad per image token"); + check(slots[0].begin == 2 && slots[1].begin == 9, "slots record where their image starts"); + + const Qwen35RopePositions p = qwen35_image_rope_positions((int) tokens.size(), slots); + // Text 0,1. Image at 2: 2 rows x 3 columns, then text resumes at 2 + 3. + // Image at 6: 2 rows x 1 column, then text resumes at 6 + 2. + check(p.temporal == std::vector({0, 1, 2, 2, 2, 2, 2, 2, 5, 6, 6, 8}), "temporal axis holds still inside an image"); + check(p.height == std::vector({0, 1, 2, 2, 2, 3, 3, 3, 5, 6, 7, 8}), "height axis counts image rows"); + check(p.width == std::vector({0, 1, 2, 3, 4, 2, 3, 4, 5, 6, 6, 8}), "width axis counts image columns"); + check(p.next == 9, "generation resumes after the widest image side"); + + std::vector chunk(4 * 3, -1); + p.fill(chunk.data(), 7, 3); + check(chunk == std::vector({2, 5, 6, 3, 5, 6, 4, 5, 6, 0, 0, 0}), "chunk fill is axis major"); + + std::vector mismatch = {1, PAD}; + check(!qwen35_expand_image_tokens(mismatch, PAD, slots, 64, error), "marker count must match image count"); + std::vector small = {1, PAD, PAD}; + check(!qwen35_expand_image_tokens(small, PAD, slots, 8, error), "expanded prompt must fit the limit"); + check(small == std::vector({1, PAD, PAD}), "a refused prompt is left unchanged"); +} + +static void test_overwrite() { + Qwen35ImageSlot slot; + slot.begin = 3; slot.columns = 2; slot.rows = 2; // tokens 3..6 + const std::vector rows = {10, 11, 20, 21, 30, 31, 40, 41}; // hidden = 2 + std::vector chunk(3 * 2, 0.0f); + qwen35_overwrite_image_rows(slot, rows.data(), 2, chunk.data(), 0, 3); // tokens 0..2 + check(chunk == std::vector(6, 0.0f), "chunk before the image is untouched"); + qwen35_overwrite_image_rows(slot, rows.data(), 2, chunk.data(), 2, 3); // tokens 2..4 + check(chunk == std::vector({0, 0, 10, 11, 20, 21}), "image head lands at its offset"); + chunk.assign(6, 0.0f); + qwen35_overwrite_image_rows(slot, rows.data(), 2, chunk.data(), 5, 3); // tokens 5..7 + check(chunk == std::vector({30, 31, 40, 41, 0, 0}), "image tail continues in the next chunk"); +} + +int main() { + test_target_size(); + test_tower_order(); + test_prompt(); + test_overwrite(); + if (failures) { std::fprintf(stderr, "%d failure(s)\n", failures); return 1; } + std::printf("OK\n"); + return 0; +} diff --git a/server/test/test_server_unit.cpp b/server/test/test_server_unit.cpp index 06e413924..7f7348eb6 100644 --- a/server/test/test_server_unit.cpp +++ b/server/test/test_server_unit.cpp @@ -20,6 +20,7 @@ #include "server/utf8_utils.h" #include "server/api_types.h" #include "server/http_server.h" +#include "server/image_input.h" #include "engine/luce_engine.h" #include "server/chat_template.h" #include "common/concurrency/seq_engine.h" @@ -60,6 +61,7 @@ #include #include #include +#include #include #include #include @@ -9359,3 +9361,206 @@ TEST_CASE(ServerUnitFixture, TEST_ASSERT(!consumed_all); TEST_ASSERT((emitted == std::vector{101, 2})); } + +namespace { +// Any text works; the transport must not know a model's marker. +constexpr char IMAGE_PLACEHOLDER[] = ""; + +json image_transport_part(const char * url = "data:image/png;base64,iVBORw0KGgo=") { + return {{"type", "image_url"}, {"image_url", {{"url", url}}}}; +} + +class OwnedImageTestPayload final : public ImagePromptPayload { +public: + explicit OwnedImageTestPayload(std::vector tokens) : tokens_(std::move(tokens)) {} + bool matches(const std::vector & tokens) const override { return tokens == tokens_; } +private: + const std::vector tokens_; +}; + +struct ImageCarrierRetryBackend : EmptySpecRetryBackend { + const ImagePromptPayload * expected = nullptr; + std::vector expected_tokens; + GenerateResult generate_impl(const GenerateRequest & req, const DaemonIO & io) override { + TEST_ASSERT(req.images.get() == expected); + TEST_ASSERT(req.prompt == expected_tokens); + TEST_ASSERT(req.images->matches(req.prompt)); + return EmptySpecRetryBackend::generate_impl(req, io); + } +}; +} + +TEST_CASE(ServerUnitFixture, test_image_extraction_normalization_preserves_interleaved_order) { + const json messages = json::array({ + {{"role", "user"}, {"content", json::array({ + {{"type", "text"}, {"text", "before "}}, image_transport_part(), + {{"type", "text"}, {"text", " between "}}, + image_transport_part("data:image/jpeg;base64,/9j/"), + {{"type", "text"}, {"text", " after"}}})}}, + {{"role", "assistant"}, {"content", "acknowledged"}}, + {{"role", "user"}, {"content", "follow-up"}} + }); + json normalized; + std::vector images; + std::string error; + TEST_ASSERT(prepare_request_images(messages, {true, true, IMAGE_PLACEHOLDER}, normalized, images, error)); + TEST_ASSERT(images.size() == 2); + TEST_ASSERT(images[0].mime_type == "image/png"); + TEST_ASSERT(images[0].bytes == std::vector({137, 80, 78, 71, 13, 10, 26, 10})); + TEST_ASSERT(images[1].mime_type == "image/jpeg"); + TEST_ASSERT(images[1].bytes == std::vector({255, 216, 255})); + ToolMemory memory; + const auto chat = normalize_chat_messages(normalized, ApiFormat::OPENAI_CHAT, memory); + TEST_ASSERT(chat.size() == 3); + TEST_ASSERT(chat[0].role == "user"); + TEST_ASSERT(chat[0].content == std::string("before ") + IMAGE_PLACEHOLDER + + " between " + IMAGE_PLACEHOLDER + " after"); + TEST_ASSERT(chat[1].content == "acknowledged" && chat[2].content == "follow-up"); + json retained = {{"messages", messages}, {"metadata", {{"image_url", "private-url"}, {"label", "keep"}}}}; + redact_image_urls(retained); + TEST_ASSERT(retained.dump().find("base64") == std::string::npos); + TEST_ASSERT(retained.dump().find("private-url") == std::string::npos); + TEST_ASSERT(retained["metadata"]["label"] == "keep"); + TEST_ASSERT(retained["messages"][0]["content"][0]["text"] == "before "); + TEST_ASSERT(messages[0]["content"][1]["image_url"]["url"] == "data:image/png;base64,iVBORw0KGgo="); +} + +TEST_CASE(ServerUnitFixture, test_image_extraction_failure_does_not_publish_partial_images) { + const json messages = json::array({{{"role", "user"}, {"content", json::array({ + image_transport_part(), image_transport_part("https://example.invalid/private.png") + })}}}); + json normalized = {{"stale", true}}; + std::vector images{{"stale", {1}}}; + std::string error; + TEST_ASSERT(!extract_chat_images(messages, IMAGE_PLACEHOLDER, normalized, images, error)); + TEST_ASSERT(normalized.is_null()); + TEST_ASSERT(images.empty()); + TEST_ASSERT(!error.empty()); + TEST_ASSERT(error.find("example.invalid") == std::string::npos); +} + +TEST_CASE(ServerUnitFixture, test_default_backend_rejects_encoded_images_without_rewriting_tokens) { + MockBackend backend; + std::vector tokens{1, 2, 3}; + ImagePromptHandle payload; + std::string error; + TEST_ASSERT(!backend.supports_images()); + TEST_ASSERT(!backend.prepare_images(tokens, {{"image/png", {137, 80, 78, 71}}}, + 8192, 32, payload, error)); + TEST_ASSERT(tokens == std::vector({1, 2, 3})); + TEST_ASSERT(!payload && !error.empty()); + TEST_ASSERT(backend.prepare_images(tokens, {}, 8192, 32, payload, error)); + TEST_ASSERT(tokens == std::vector({1, 2, 3})); +} + +TEST_CASE(ServerUnitFixture, test_image_binding_survives_request_copies_and_common_ar_retry) { + static_assert(std::is_const_v); + GenerateRequest request; + std::weak_ptr lifetime; + { + ParsedRequest parsed; + parsed.prompt_tokens = {1, 129280, 129281, 2}; + parsed.images = std::make_shared(parsed.prompt_tokens); + lifetime = parsed.images; + ParsedRequest queued = parsed; + parsed.prompt_tokens.clear(); + parsed.images.reset(); + request.prompt = queued.prompt_tokens; + request.images = queued.images; + } + TEST_ASSERT(!lifetime.expired()); + auto changed = request.prompt; + changed[1] += 1; + TEST_ASSERT(!request.images->matches(changed)); + TEST_ASSERT(request.images->matches(request.prompt)); + ImageCarrierRetryBackend backend; + backend.expected = request.images.get(); + backend.expected_tokens = request.prompt; + request.n_gen = 1; + DaemonIO io; + TEST_ASSERT(backend.generate(request, io).ok()); + TEST_ASSERT(backend.generate_calls == 2 && backend.generate_saw_force_ar); + request.images.reset(); + TEST_ASSERT(lifetime.expired()); +} + +TEST_CASE(ServerUnitFixture, test_http_image_policy_rejects_unconsumed_images_including_mixed_valid) { + const json image = image_transport_part(); + const std::vector malformed_messages = { + {{"role", "user"}, {"content", image}}, + {{"role", "user"}, {"content", json::array({ + {{"type", "text"}, {"text", "visible"}, {"image_url", image["image_url"]}} + })}}, + {{"role", "user"}, {"content", json::array({ + {{"type", "container"}, {"nested", image}} + })}}, + {{"role", "user"}, {"content", "visible"}, {"attachment", image}}, + {{"role", "user"}, {"content", json::array({ + {{"type", "input_image"}, {"image_url", image["image_url"]}} + })}} + }; + for (const auto & malformed : malformed_messages) { + for (bool mixed : {false, true}) { + json messages = json::array(); + if (mixed) messages.push_back({{"role", "user"}, {"content", json::array({image})}}); + messages.push_back(malformed); + json normalized = "stale"; + std::vector images{{"stale", {1}}}; + std::string error; + TEST_ASSERT(!prepare_request_images(messages, {true, true, IMAGE_PLACEHOLDER}, normalized, images, error)); + TEST_ASSERT(normalized.is_null() && images.empty()); + TEST_ASSERT(!error.empty()); + TEST_ASSERT(error.find("base64") == std::string::npos); + } + } +} + +TEST_CASE(ServerUnitFixture, test_http_image_policy_requires_chat_endpoint_and_effective_capability) { + const json messages = json::array({{{"role", "user"}, {"content", json::array({image_transport_part()})}}}); + { + json normalized = "stale"; + std::vector images{{"stale", {1}}}; + std::string error; + TEST_ASSERT(!prepare_request_images(messages, {false, true, IMAGE_PLACEHOLDER}, normalized, images, error)); + TEST_ASSERT(normalized.is_null() && images.empty()); + TEST_ASSERT(!error.empty()); + } + // Without image capability the messages pass through untouched. + for (ImageRequestPolicy policy : { + ImageRequestPolicy{true, false, IMAGE_PLACEHOLDER}, + ImageRequestPolicy{false, false, IMAGE_PLACEHOLDER}}) { + json normalized = "stale"; + std::vector images{{"stale", {1}}}; + std::string error; + TEST_ASSERT(prepare_request_images(messages, policy, normalized, images, error)); + TEST_ASSERT(normalized == messages && images.empty()); + } + json normalized; + std::vector images; + std::string error; + TEST_ASSERT(prepare_request_images(messages, {true, true, IMAGE_PLACEHOLDER}, normalized, images, error)); + TEST_ASSERT(images.size() == 1); +} + +TEST_CASE(ServerUnitFixture, test_http_image_policy_preserves_text_without_image_capability) { + const json messages = json::array({ + {{"role", "system"}, {"content", "instructions"}}, + {{"role", "user"}, {"content", json::array({{{"type", "text"}, {"text", "ordinary text"}}})}} + }); + for (ImageRequestPolicy policy : { + ImageRequestPolicy{true, false, IMAGE_PLACEHOLDER}, + ImageRequestPolicy{false, false, IMAGE_PLACEHOLDER}, + ImageRequestPolicy{true, true, IMAGE_PLACEHOLDER}}) { + json normalized; + std::vector images; + std::string error; + TEST_ASSERT(prepare_request_images(messages, policy, normalized, images, error)); + TEST_ASSERT(normalized == messages && images.empty()); + } + json normalized; + std::vector images; + std::string error; + const json forged = json::array({{{"role", "user"}, {"content", IMAGE_PLACEHOLDER}}}); + TEST_ASSERT(!prepare_request_images(forged, {true, true, IMAGE_PLACEHOLDER}, normalized, images, error)); + TEST_ASSERT(normalized.is_null() && images.empty()); +} diff --git a/server/tests/test_deepseek4_unit.cpp b/server/tests/test_deepseek4_unit.cpp index 7730bd769..f1f671ec3 100644 --- a/server/tests/test_deepseek4_unit.cpp +++ b/server/tests/test_deepseek4_unit.cpp @@ -1,6 +1,8 @@ #include "ggml.h" #include "ggml-alloc.h" #include "ggml-backend.h" +#include "../deps/llama.cpp/ggml/src/ggml-backend-impl.h" +#include "deepseek4/deepseek4_image_admission.h" #include "ggml-cpu.h" #include "../deps/llama.cpp/ggml/src/ggml-cuda/ds4-causal.h" #if defined(GGML_USE_HIP) @@ -234,6 +236,15 @@ struct DeepSeek4FixtureOptions { gguf_type compress_ratios_type = GGUF_TYPE_UINT32; int32_t eos_id = -1; int32_t eot_id = -1; + uint32_t block_count = 43; + bool image_biases = false; + int missing_image_bias = -1; + int malformed_image_bias = -1; + ggml_type image_bias_type = GGML_TYPE_F32; + int image_bias_width = 256; + int image_bias_rows = 1; + bool add_mtp_image_bias = false; + bool llama_cpp_image_bias_names = false; // "blk.N.exp_probs_b_vl.bias" }; static std::string make_temp_gguf_path(const char * prefix) { @@ -249,7 +260,7 @@ static std::string make_temp_gguf_path(const char * prefix) { static std::string write_deepseek4_loader_fixture(const DeepSeek4FixtureOptions & opts) { gguf_context * g = gguf_init_empty(); gguf_set_val_str(g, "general.architecture", "deepseek4"); - gguf_set_val_u32(g, "deepseek4.block_count", 43); + gguf_set_val_u32(g, "deepseek4.block_count", opts.block_count); gguf_set_val_u32(g, "deepseek4.embedding_length", 4096); if (opts.include_vocab_size) { gguf_set_val_u32(g, "deepseek4.vocab_size", opts.vocab_size); @@ -304,9 +315,31 @@ static std::string write_deepseek4_loader_fixture(const DeepSeek4FixtureOptions gguf_set_val_u32(g, "tokenizer.ggml.eot_token_id", (uint32_t)opts.eot_id); } + ggml_context * tensor_ctx = nullptr; + if (opts.image_biases) { + tensor_ctx = ggml_init({1u << 20, nullptr, false}); + for (int layer = 0; layer < (opts.add_mtp_image_bias ? 44 : 43); ++layer) { + if (layer == opts.missing_image_bias) continue; + const bool malformed = layer == opts.malformed_image_bias; + ggml_tensor * bias = ggml_new_tensor_2d(tensor_ctx, + malformed ? opts.image_bias_type : GGML_TYPE_F32, + malformed ? opts.image_bias_width : 256, + malformed ? opts.image_bias_rows : 1); + const std::string name = opts.llama_cpp_image_bias_names + ? "blk." + std::to_string(layer) + ".exp_probs_b_vl.bias" + : "layers." + std::to_string(layer) + ".ffn.gate.bias_vl"; + ggml_set_name(bias, name.c_str()); + std::memset(bias->data, 0, ggml_nbytes(bias)); + if (bias->type == GGML_TYPE_F32) { + std::fill_n(static_cast(bias->data), ggml_nelements(bias), float(layer + 1)); + } + gguf_add_tensor(g, bias); + } + } const std::string path = make_temp_gguf_path("fixture"); gguf_write_to_file(g, path.c_str(), /*only_meta=*/false); gguf_free(g); + if (tensor_ctx) ggml_free(tensor_ctx); return path; } @@ -1531,6 +1564,323 @@ struct ScopedEnvVar { std::string old_value; }; +// Metadata-only device: every allocation/graph callback is a test failure. +struct ImageAdmissionFakeOwner { + ggml_backend_buffer_type buft{}; + ggml_backend_device device{}; + ggml_backend backend{}; + size_t alignment = 128; + size_t padding = 64; + size_t maximum = SIZE_MAX; + size_t forced_allocation = 0; + size_t free_bytes = 8ULL * 1024 * 1024 * 1024; + size_t total_bytes = 8ULL * 1024 * 1024 * 1024; + size_t queries = 0; + size_t allocation_calls = 0; + size_t graph_calls = 0; + + ImageAdmissionFakeOwner() { + buft.context = this; + buft.device = &device; + buft.iface.get_name = [](ggml_backend_buffer_type_t) { return "image-admission-fake"; }; + buft.iface.get_alignment = [](ggml_backend_buffer_type_t b) { + return static_cast(b->context)->alignment; + }; + buft.iface.get_max_size = [](ggml_backend_buffer_type_t b) { + return static_cast(b->context)->maximum; + }; + buft.iface.get_alloc_size = [](ggml_backend_buffer_type_t b, const ggml_tensor * t) { + auto & owner = *static_cast(b->context); + ++owner.queries; + return owner.forced_allocation ? owner.forced_allocation : ggml_nbytes(t) + owner.padding; + }; + buft.iface.alloc_buffer = [](ggml_backend_buffer_type_t b, size_t) -> ggml_backend_buffer_t { + ++static_cast(b->context)->allocation_calls; + TEST_ASSERT_MSG(false, "admission must not allocate a device buffer"); + return nullptr; + }; + device.context = this; + device.iface.get_type = [](ggml_backend_dev_t) { return GGML_BACKEND_DEVICE_TYPE_GPU; }; + device.iface.get_buffer_type = [](ggml_backend_dev_t d) { + return &static_cast(d->context)->buft; + }; + device.iface.get_memory = [](ggml_backend_dev_t d, size_t * free, size_t * total) { + const auto & owner = *static_cast(d->context); + *free = owner.free_bytes; + *total = owner.total_bytes; + }; + backend.device = &device; + backend.context = this; + backend.iface.graph_compute = [](ggml_backend_t b, ggml_cgraph *) { + ++static_cast(b->context)->graph_calls; + TEST_ASSERT_MSG(false, "admission must not execute a graph"); + return GGML_STATUS_FAILED; + }; + } +}; + +static void test_image_storage_admission_metadata() { + std::fprintf(stderr, "test_image_storage_admission_metadata..."); + using namespace luce::vision; + ScopedEnvVar duplicate_env("LUCE_MOE_DUPLICATE_HOT_ON_COLD"); + ScopedEnvVar decode_env("LUCE_DS4_DECODE_ALL_COLD"); + unsetenv("LUCE_MOE_DUPLICATE_HOT_ON_COLD"); + unsetenv("LUCE_DS4_DECODE_ALL_COLD"); + ggml_init_params params{}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + TEST_ASSERT(ctx != nullptr); + if (!ctx) return; + DeepSeek4Weights weights; + weights.n_layer = 1; + weights.n_expert = 4; + weights.n_expert_used = 2; + weights.n_embd = 128; + weights.n_ff_exp = 128; + weights.layers.resize(1); + auto & layer = weights.layers[0]; + layer.ffn_gate_exps = ggml_new_tensor_3d(ctx, GGML_TYPE_Q2_1_ROCMFP2_MIX, 128, 128, 4); + layer.ffn_up_exps = ggml_new_tensor_3d(ctx, GGML_TYPE_Q2_1_ROCMFP2_MIX, 128, 128, 4); + layer.ffn_down_exps = ggml_new_tensor_3d(ctx, GGML_TYPE_Q3_1_ROCMFP3_MIX, 128, 128, 4); + MoeHybridConfig config; + config.n_layer = 1; + config.n_expert = 4; + config.n_expert_used = 2; + config.n_embd = 128; + config.n_ff_exp = 128; + config.cold_expert_backend = MoeHybridColdBackend::Gpu; + MoeHybridPlacement placement; + placement.n_layer = 1; + placement.n_expert = 4; + placement.n_expert_used = 2; + placement.total_hot = 1; + placement.hot_counts = {1}; + placement.hot_expert_ids = {{2}}; + ImageAdmissionFakeOwner hot, cold; + cold.alignment = 256; + cold.padding = 257; + std::string error; + ImageStorageEstimate initial; + TEST_ASSERT(estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, initial, error)); + // Pinned qtype-106 and105 formats occupy10 and14 bytes per32 weights. + // Three128x128 surfaces occupy5120+5120+7168 bytes per expert. + TEST_ASSERT(initial.hot_payload_bytes == 17408); + TEST_ASSERT(initial.cold_payload_bytes == 3 * 17408); + TEST_ASSERT(initial.hot_allocation_bytes == 17408 + 3 * 128); + TEST_ASSERT(initial.cold_allocation_bytes == 3 * 17408 + 3 * 512); + TEST_ASSERT(initial.hot_mix_table_bytes == 17 + 17 + 33); + TEST_ASSERT(initial.cold_mix_table_bytes == 3 * (17 + 17 + 33)); + TEST_ASSERT(initial.mix_device_allocation_count == 12); + TEST_ASSERT(initial.largest_copy_bytes == 3 * 7168); + TEST_ASSERT(initial.host_copy_peak_bytes == 3 * initial.largest_copy_bytes); + TEST_ASSERT(initial.host_mix_payload_peak_bytes == 34 * 4 + 33 * 3 + 4); + TEST_ASSERT(initial.hot_buffer_count == 1 && initial.cold_buffer_count == 1); + TEST_ASSERT(hot.queries == 3 && cold.queries == 3); + + // Changing the actual allocator's maximum splits buffers without losing + // padding charges or pretending the whole layer must fit one allocation. + hot.maximum = 10000; + cold.maximum = 23000; + ImageStorageEstimate split; + TEST_ASSERT(estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, split, error)); + TEST_ASSERT(split.hot_buffer_count == 3 && split.cold_buffer_count == 3); + TEST_ASSERT(split.hot_allocation_bytes == initial.hot_allocation_bytes); + TEST_ASSERT(split.cold_allocation_bytes == initial.cold_allocation_bytes); + cold.maximum = 22015; // one byte below the padded down-expert allocation + TEST_ASSERT(!estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, split, error)); + hot.maximum = cold.maximum = SIZE_MAX; + cold.forced_allocation = SIZE_MAX; + TEST_ASSERT(!estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, split, error)); + TEST_ASSERT(error.find("allocation size") != std::string::npos); + cold.forced_allocation = 0; + + setenv("LUCE_MOE_DUPLICATE_HOT_ON_COLD", "1", 1); + ImageStorageEstimate duplicated; + TEST_ASSERT(estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, true, duplicated, error)); + TEST_ASSERT(duplicated.hot_payload_bytes == initial.hot_payload_bytes); + TEST_ASSERT(duplicated.cold_payload_bytes == initial.hot_payload_bytes + initial.cold_payload_bytes); + TEST_ASSERT(duplicated.cold_mix_table_bytes == 4 * (17 + 17 + 33)); + TEST_ASSERT(!estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, duplicated, error)); + unsetenv("LUCE_MOE_DUPLICATE_HOT_ON_COLD"); + setenv("LUCE_DS4_DECODE_ALL_COLD", "1", 1); + TEST_ASSERT(!estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, split, error)); + unsetenv("LUCE_DS4_DECODE_ALL_COLD"); + + placement.total_hot = 0; + placement.hot_counts = {0}; + placement.hot_expert_ids = {{}}; + TEST_ASSERT(estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, split, error)); + TEST_ASSERT(split.hot_payload_bytes == 0 && split.hot_mix_table_bytes == 0); + TEST_ASSERT(split.cold_payload_bytes == initial.hot_payload_bytes + initial.cold_payload_bytes); + TEST_ASSERT(split.cold_mix_table_bytes == 4 * 67); + placement.total_hot = 1; + placement.hot_counts = {1}; + placement.hot_expert_ids = {{2}}; + const int64_t original_experts = layer.ffn_gate_exps->ne[2]; + layer.ffn_gate_exps->ne[2] = 3; + TEST_ASSERT(!estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, split, error)); + layer.ffn_gate_exps->ne[2] = original_experts; + const size_t original_stride = layer.ffn_gate_exps->nb[2]; + ++layer.ffn_gate_exps->nb[2]; + TEST_ASSERT(!estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, split, error)); + layer.ffn_gate_exps->nb[2] = original_stride; + TEST_ASSERT(!estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &hot.backend, false, split, error)); + config.materialize_cold_experts = false; + TEST_ASSERT(!estimate_deepseek4_image_storage(weights, placement, config, + &hot.backend, &cold.backend, false, split, error)); + config.materialize_cold_experts = true; + + // Actual wrapper sees the fake device's exhausted snapshot. It cannot pass + // regardless of the machine's memory; no positive case reads /proc. The + // fake is dedicated memory: a host-shared device is also credited with the + // GPU driver's page pool, which depends on the machine running the test. + ImageAdmissionReserves reserves; + reserves.primary_domain = ImageMemoryDomain::Dedicated; + reserves.cold_domain = ImageMemoryDomain::Dedicated; + reserves.cold_runtime_reservation_bytes = 2ULL * 1024 * 1024 * 1024; + reserves.host_request_bytes = 1024 * 1024; + reserves.host_loader_overhead_bytes = 1024 * 1024; + cold.free_bytes = 0; + ImageAdmissionReport report; + TEST_ASSERT(!check_deepseek4_image_admission(weights, placement, config, + &hot.backend, &cold.backend, reserves, report, error)); + TEST_ASSERT(report.storage_estimated && !report.known_charges_fit); + TEST_ASSERT(report.cold_free_bytes == 0); + TEST_ASSERT(!check_deepseek4_image_runtime_admission(config, + &hot.backend, &cold.backend, reserves, report, error)); + TEST_ASSERT(report.storage.hot_allocation_bytes == 0 && report.storage.cold_allocation_bytes == 0); + TEST_ASSERT(report.cold_required_bytes == reserves.cold_runtime_reservation_bytes); + TEST_ASSERT(!check_deepseek4_image_host_preparation(0, error)); + TEST_ASSERT(!check_deepseek4_image_host_preparation(std::numeric_limits::max(), error)); + TEST_ASSERT(hot.allocation_calls == 0 && cold.allocation_calls == 0); + TEST_ASSERT(hot.graph_calls == 0 && cold.graph_calls == 0); + ggml_free(ctx); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + +static void test_image_admission_resource_snapshots() { + std::fprintf(stderr, "test_image_admission_resource_snapshots..."); + using namespace luce::vision; + ImageStorageEstimate storage; + storage.hot_allocation_bytes = 100; + storage.cold_allocation_bytes = 300; + storage.hot_mix_table_bytes = 10; + storage.cold_mix_table_bytes = 30; + storage.host_copy_peak_bytes = 50; + storage.host_mix_payload_peak_bytes = 5; + ImageAdmissionReserves reserves; + reserves.primary_domain = ImageMemoryDomain::Dedicated; + reserves.cold_domain = ImageMemoryDomain::HostShared; + reserves.primary_future_bytes = 20; + reserves.cold_future_bytes = 40; + reserves.cold_runtime_reservation_bytes = 200; + reserves.host_loader_overhead_bytes = 15; + reserves.host_request_bytes = 100; + reserves.host_runtime_bytes = 7; + ImageMemorySnapshot snapshot{130, 570, 677}; + ImageAdmissionReport report; + std::string error; + const auto assess = [&]() { + return assess_deepseek4_image_admission(storage, 150, reserves, snapshot, report, error); + }; + TEST_ASSERT(assess()); + TEST_ASSERT(report.primary_required_bytes == 130 && report.cold_required_bytes == 570); + TEST_ASSERT(report.host_required_bytes == 677); // max(load70,request100)+runtime7+UMA570 + --snapshot.host_available_bytes; + TEST_ASSERT(!assess()); // independent device checks would both pass + TEST_ASSERT(!report.known_charges_fit); + snapshot.host_available_bytes = 677; + --snapshot.cold_free_bytes; + TEST_ASSERT(!assess()); + snapshot.cold_free_bytes = 570; + --snapshot.primary_free_bytes; + TEST_ASSERT(!assess()); + snapshot.primary_free_bytes = 130; + reserves.cold_domain = ImageMemoryDomain::Dedicated; + snapshot.host_available_bytes = 107; + TEST_ASSERT(assess()); // dedicated VRAM must not also consume host admission + reserves.primary_domain = ImageMemoryDomain::HostShared; + reserves.cold_domain = ImageMemoryDomain::HostShared; + snapshot.host_available_bytes = 807; + TEST_ASSERT(assess()); + TEST_ASSERT(report.host_required_bytes == 807); + --snapshot.host_available_bytes; + TEST_ASSERT(!assess()); + reserves.primary_domain = ImageMemoryDomain::Unknown; + TEST_ASSERT(!assess()); + reserves.primary_domain = ImageMemoryDomain::Dedicated; + snapshot.host_available_bytes = 677; + reserves.host_request_bytes = 60; + TEST_ASSERT(assess()); + TEST_ASSERT(report.host_required_bytes == 647); // load70 exceeds request60 + reserves.host_request_bytes = 100; + reserves.cold_runtime_reservation_bytes = 149; + TEST_ASSERT(!assess()); + TEST_ASSERT(report.known_charges_fit); // rejection is missing runtime headroom, not physical exhaustion + reserves.cold_runtime_reservation_bytes = 200; + reserves.host_request_bytes = 0; + TEST_ASSERT(!assess()); + reserves.host_request_bytes = 100; + reserves.host_loader_overhead_bytes = 0; + TEST_ASSERT(!assess()); + reserves.host_loader_overhead_bytes = 15; + + // After startup materializes the experts/tables, their bytes disappear + // from free memory and from new allocations together. Retained KV/draft + // snapshots added later only shrink the next free-memory snapshot. + const ImageStorageEstimate resident_storage{}; + ImageAdmissionReserves runtime_reserves = reserves; + runtime_reserves.host_loader_overhead_bytes = 0; + ImageMemorySnapshot runtime_snapshot{20, 240, 347}; + const auto runtime_assess = [&]() { + return assess_deepseek4_image_admission(resident_storage, 150, + runtime_reserves, runtime_snapshot, report, error); + }; + TEST_ASSERT(runtime_assess()); + TEST_ASSERT(report.primary_required_bytes == 20 && report.cold_required_bytes == 240); + TEST_ASSERT(report.host_required_bytes == 347); + --runtime_snapshot.primary_free_bytes; + TEST_ASSERT(!runtime_assess()); + runtime_snapshot.primary_free_bytes = 20; + --runtime_snapshot.cold_free_bytes; + TEST_ASSERT(!runtime_assess()); + runtime_snapshot.cold_free_bytes = 240; + --runtime_snapshot.host_available_bytes; + TEST_ASSERT(!runtime_assess()); + + constexpr uint64_t max = std::numeric_limits::max(); + snapshot = {max, max, max}; + storage.hot_allocation_bytes = max; + TEST_ASSERT(!assess()); + TEST_ASSERT(error.find("overflow") != std::string::npos); + storage.hot_allocation_bytes = 100; + storage.host_copy_peak_bytes = max; + TEST_ASSERT(!assess()); + TEST_ASSERT(error.find("overflow") != std::string::npos); + storage.host_copy_peak_bytes = 50; + reserves.host_request_bytes = max - 100; + TEST_ASSERT(!assess()); // UMA sum overflows even though separate device budgets fit + TEST_ASSERT(error.find("overflow") != std::string::npos); + reserves.host_request_bytes = 100; + TEST_ASSERT(assess()); + TEST_ASSERT(assess_deepseek4_image_admission(report.storage, 150, reserves, + snapshot, report, error)); // report may safely be reused without losing its estimate + TEST_ASSERT(report.primary_required_bytes == 130); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + static void test_verify_raw_mask_spans() { std::fprintf(stderr, " test_verify_raw_mask_spans ..."); int cases = 0; @@ -1918,8 +2268,9 @@ static void test_loader_rejects_missing_required_metadata(ggml_backend_t backend DeepSeek4Weights weights; const bool ok = load_deepseek4_gguf(path, backend, weights); TEST_ASSERT(!ok); + // The fixture has no tokenizer token list either, so no size can be derived. TEST_ASSERT_MSG(std::string(luce_last_error()).find( - "missing required key: deepseek4.vocab_size") != std::string::npos, + "no vocabulary size") != std::string::npos, luce_last_error()); free_deepseek4_weights(weights); unlink(path.c_str()); @@ -1956,7 +2307,7 @@ static void test_loader_rejects_zero_vocab_size(ggml_backend_t backend) { const bool ok = load_deepseek4_gguf(path, backend, weights); TEST_ASSERT(!ok); TEST_ASSERT_MSG(std::string(luce_last_error()).find( - "deepseek4.vocab_size must be > 0") != std::string::npos, + "no vocabulary size") != std::string::npos, luce_last_error()); free_deepseek4_weights(weights); unlink(path.c_str()); @@ -2018,6 +2369,221 @@ static void test_loader_rejects_truncated_tensor_data(ggml_backend_t backend) { std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); } + +static void test_image_bias_loader_opt_in_contract(ggml_backend_t backend) { + std::fprintf(stderr, " test_image_bias_loader_opt_in_contract ..."); + DeepSeek4FixtureOptions valid; + valid.vocab_size = 129280; + valid.image_biases = true; + valid.add_mtp_image_bias = true; + // Both spellings load: the source checkpoint's and llama.cpp's. + for (bool llama_cpp_names : {false, true}) { + valid.llama_cpp_image_bias_names = llama_cpp_names; + const std::string path = write_deepseek4_loader_fixture(valid); + for (bool enabled : {false, true}) { + TargetLoadPlan plan; + plan.load_ds4_image_bias = enabled; + DeepSeek4Weights weights; + const bool ok = load_deepseek4_gguf_partial(path, backend, plan, weights); + TEST_ASSERT_MSG(ok, luce_last_error()); + if (ok) { + TEST_ASSERT(weights.layers.size() == 43); + for (size_t i = 0; i < weights.layers.size(); ++i) { + const auto bias = weights.layers[i].ffn_gate_bias_vl; + TEST_ASSERT(bool(bias) == enabled); + if (bias) { + TEST_ASSERT(bias->type == GGML_TYPE_F32 && ggml_nelements(bias) == 256); + std::vector values(256); + ggml_backend_tensor_get(bias, values.data(), 0, values.size() * sizeof(float)); + TEST_ASSERT(std::all_of(values.begin(), values.end(), + [i](float value) { return value == float(i + 1); })); + } + } + if (weights.ctx) { + const auto mtp = ggml_get_tensor(weights.ctx, llama_cpp_names + ? "blk.43.exp_probs_b_vl.bias" : "layers.43.ffn.gate.bias_vl"); + TEST_ASSERT(mtp == nullptr || (mtp->buffer == nullptr && mtp->data == nullptr)); + } + } + free_deepseek4_weights(weights); + } + unlink(path.c_str()); + } + valid.llama_cpp_image_bias_names = false; + const std::string path = write_deepseek4_loader_fixture(valid); + for (int boundary : {0, 1}) { + TargetLoadPlan plan; + plan.load_ds4_image_bias = true; + if (boundary == 0) plan.layer_begin = 1; + else plan.layer_end = 42; + DeepSeek4Weights weights; + TEST_ASSERT(!load_deepseek4_gguf_partial(path, backend, plan, weights)); + TEST_ASSERT_MSG(std::string(luce_last_error()).find("one F32[n_expert] image router bias per layer") != std::string::npos, + luce_last_error()); + TEST_ASSERT(weights.ctx == nullptr && weights.buf == nullptr); + free_deepseek4_weights(weights); + } + unlink(path.c_str()); + + std::vector invalid; + auto missing_all = valid; + missing_all.image_biases = false; + invalid.push_back(missing_all); + for (int layer : {0, 21, 42}) { + auto missing = valid; + missing.missing_image_bias = layer; + invalid.push_back(missing); + } + auto wrong_type = valid; + wrong_type.malformed_image_bias = 21; + wrong_type.image_bias_type = GGML_TYPE_F16; + invalid.push_back(wrong_type); + auto wrong_width = valid; + wrong_width.malformed_image_bias = 42; + wrong_width.image_bias_width = 255; + invalid.push_back(wrong_width); + auto matrix = valid; + matrix.malformed_image_bias = 0; + matrix.image_bias_width = 128; + matrix.image_bias_rows = 2; + invalid.push_back(matrix); + // Decoder width and vocabulary are checked against the projector's own + // metadata when it loads, not here. + for (const auto & options : invalid) { + const std::string bad_path = write_deepseek4_loader_fixture(options); + TargetLoadPlan plan; + plan.load_ds4_image_bias = true; + DeepSeek4Weights weights; + TEST_ASSERT(!load_deepseek4_gguf_partial(bad_path, backend, plan, weights)); + TEST_ASSERT_MSG(std::string(luce_last_error()).find("one F32[n_expert] image router bias per layer") != std::string::npos, + luce_last_error()); + TEST_ASSERT(weights.ctx == nullptr && weights.buf == nullptr && weights.dense_split_buf == nullptr); + free_deepseek4_weights(weights); + plan.load_ds4_image_bias = false; + TEST_ASSERT_MSG(load_deepseek4_gguf_partial(bad_path, backend, plan, weights), luce_last_error()); + for (const auto & layer : weights.layers) TEST_ASSERT(layer.ffn_gate_bias_vl == nullptr); + free_deepseek4_weights(weights); + unlink(bad_path.c_str()); + } + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + +static void test_image_batch_admission_before_execution(ggml_backend_t backend) { + std::fprintf(stderr, " test_image_batch_admission_before_execution ..."); + ggml_context * ctx = ggml_init({1u << 20, nullptr, false}); + ggml_tensor * bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 256); + ggml_tensor * state = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); + static_cast(state->data)[0] = 123.0f; + DeepSeek4Weights weights; + weights.n_vocab = 129280; + weights.n_layer = 43; + weights.moe_hybrid = true; + weights.layers.resize(43); + weights.compress_ratios.resize(43); + DeepSeek4Cache cache; + cache.max_ctx = 32; + cache.cur_pos = 17; + cache.prefill_mode = PrefillAttentionMode::Sparse; + cache.layers.resize(43); + MoeHybridStorage hybrid; + hybrid.layers.resize(43); + hybrid.cold_backend_kind = MoeHybridColdBackend::Gpu; + hybrid.cold_backend = backend; + for (int i = 0; i < 43; ++i) { + weights.compress_ratios[size_t(i)] = i < 2 ? 0 : i % 2 == 0 ? 4 : 128; + auto & layer = weights.layers[size_t(i)]; + auto & layer_cache = cache.layers[size_t(i)]; + layer.ffn_gate_bias_vl = bias; + layer.attn_compressor_ape = layer.attn_compressor_kv = layer.attn_compressor_gate = layer.attn_compressor_norm = state; + layer.indexer_compressor_ape = layer.indexer_compressor_kv = layer.indexer_compressor_gate = layer.indexer_compressor_norm = state; + layer_cache.raw_kv = layer_cache.comp_kv = layer_cache.index_comp_kv = state; + layer_cache.attn_compressor.state_kv = layer_cache.attn_compressor.state_score = state; + layer_cache.indexer_compressor.state_kv = layer_cache.indexer_compressor.state_score = state; + } + const luce::vision::TokenSpan span{1, 2, 5, 6}; + const luce::vision::ImageSpanView spans{&span, 1}; + std::vector tokens{7, 129280, 129281, 129282, 129283, 129284, 8}; + bool has_images = false; + std::string error; + auto validate = [&]() { + error.clear(); + return deepseek4_validate_image_batch(weights, cache, &hybrid, tokens.data(), + int(tokens.size()), 0, spans, has_images, error); + }; + TEST_ASSERT(validate() && has_images); + for (size_t position : {size_t(0), size_t(1), size_t(5), size_t(6)}) { + const int32_t original = tokens[position]; + tokens[position] = position == 0 || position == 6 ? 129280 : 7; + TEST_ASSERT(!validate()); + TEST_ASSERT(error.find("token IDs") != std::string::npos); + tokens[position] = original; + } + tokens[2] = 129285; + TEST_ASSERT(!validate()); + tokens[2] = 129281; + TEST_ASSERT(!deepseek4_validate_image_batch(weights, cache, &hybrid, tokens.data(), + 5, 0, spans, has_images, error)); + TEST_ASSERT(!deepseek4_validate_image_batch(weights, cache, &hybrid, tokens.data() + 2, + 5, 2, spans, has_images, error)); + TEST_ASSERT(!deepseek4_validate_image_batch(weights, cache, &hybrid, nullptr, + 7, 0, spans, has_images, error)); + const auto check_missing_tensor = [&](ggml_tensor * & tensor) { + ggml_tensor * saved = tensor; + tensor = nullptr; + TEST_ASSERT(!validate()); + tensor = saved; + TEST_ASSERT(validate()); + }; + for (int i : {0, 2, 3, 42}) { + auto & layer = weights.layers[size_t(i)]; + auto & layer_cache = cache.layers[size_t(i)]; + check_missing_tensor(layer.ffn_gate_bias_vl); + check_missing_tensor(layer_cache.raw_kv); + if (i >= 2) { + check_missing_tensor(layer.attn_compressor_gate); + check_missing_tensor(layer_cache.attn_compressor.state_score); + } + if (i >= 2 && i % 2 == 0) { + check_missing_tensor(layer.indexer_compressor_kv); + check_missing_tensor(layer_cache.indexer_compressor.state_kv); + } + } + weights.layers[42].ffn_gate_bias_vl = ggml_new_tensor_1d(ctx, GGML_TYPE_F16, 256); + TEST_ASSERT(!validate()); + weights.layers[42].ffn_gate_bias_vl = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 256, 2); + TEST_ASSERT(!validate()); + weights.layers[42].ffn_gate_bias_vl = bias; + TEST_ASSERT(validate()); + hybrid.materialized_cold_experts = false; + TEST_ASSERT(!validate()); + hybrid.materialized_cold_experts = true; + cache.layers.pop_back(); + TEST_ASSERT(!validate()); + TEST_ASSERT(cache.cur_pos == 17 && static_cast(state->data)[0] == 123.0f); + const luce::vision::TokenSpan invalid_span{1, 2, 7, 6}; + TEST_ASSERT(!deepseek4_validate_image_batch(weights, cache, &hybrid, tokens.data(), + 7, 0, {&invalid_span, 1}, has_images, error)); + const int32_t text[] = {1, 2}; + TEST_ASSERT(deepseek4_validate_image_batch(weights, cache, nullptr, text, 2, 6, + spans, has_images, error) && !has_images); + // One GPU holding the whole model has no second expert owner to check. + // (Dense weights here are unset, so the per-layer tensor checks still apply.) + weights.moe_hybrid = false; + cache.layers.resize(43); + for (auto & layer_cache : cache.layers) { + layer_cache.raw_kv = layer_cache.comp_kv = layer_cache.index_comp_kv = state; + layer_cache.attn_compressor.state_kv = layer_cache.attn_compressor.state_score = state; + layer_cache.indexer_compressor.state_kv = layer_cache.indexer_compressor.state_score = state; + } + TEST_ASSERT(deepseek4_validate_image_batch(weights, cache, nullptr, tokens.data(), + int(tokens.size()), 0, spans, has_images, error) && has_images); + cache.prefill_mode = PrefillAttentionMode::Exact; + TEST_ASSERT(!deepseek4_validate_image_batch(weights, cache, nullptr, tokens.data(), + int(tokens.size()), 0, spans, has_images, error)); + ggml_free(ctx); + std::fprintf(stderr, g_failures ? " done\n" : " ok\n"); +} + static void test_dspark_loader_contract_and_bounds(ggml_backend_t backend) { std::fprintf(stderr, " test_dspark_loader_contract_and_bounds ..."); @@ -7273,6 +7839,10 @@ int main(int argc, char ** argv) { test_loader_rejects_zero_vocab_size(backend); test_loader_reads_tokenizer_special_ids(backend); test_loader_rejects_truncated_tensor_data(backend); + test_image_bias_loader_opt_in_contract(backend); + test_image_batch_admission_before_execution(backend); + test_image_storage_admission_metadata(); + test_image_admission_resource_snapshots(); test_dspark_loader_contract_and_bounds(backend); test_dspark_confidence_uses_separate_hidden(backend); test_safe_compressor_batch_tokens(); diff --git a/server/tests/test_export_ds4v_mmproj.py b/server/tests/test_export_ds4v_mmproj.py new file mode 100644 index 000000000..1304de212 --- /dev/null +++ b/server/tests/test_export_ds4v_mmproj.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +import importlib.util +import json +import struct +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +REPO_ROOT = Path(__file__).resolve().parents[2] +EXPORTER_PATH = REPO_ROOT / "server" / "tools" / "export_ds4v_mmproj.py" +SPEC = importlib.util.spec_from_file_location("export_ds4v_mmproj", EXPORTER_PATH) +assert SPEC is not None and SPEC.loader is not None +exporter = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = exporter +SPEC.loader.exec_module(exporter) + + +TEST_SHAPES = { + "aligner.tiny": (2,), + "image_start": (3,), + "vision.tiny": (2, 2), +} + + +def exact_config(): + return dict(exporter.EXPECTED_CONFIG) + + +def write_json(path, value): + path.write_text(json.dumps(value), encoding="utf-8") + + +def write_safetensors(path, entries, payload): + raw_header = json.dumps(entries, separators=(",", ":")).encode("utf-8") + path.write_bytes(struct.pack(" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using json = nlohmann::json; +namespace fs = std::filesystem; + +namespace { + +constexpr uint32_t kQtypeP4Mix = 105; +constexpr uint32_t kQtypeGuMix = 106; +constexpr uint32_t kCodebooks = 2; +constexpr uint32_t kBlock = 32; +constexpr uint32_t kP4Levels = 8; +constexpr uint32_t kGuLevels = 4; +constexpr uint32_t kAlignment = 32; +constexpr size_t kCopyChunk = 8u * 1024u * 1024u; + +[[noreturn]] void fail(const std::string & message) { + throw std::runtime_error(message); +} + +// Worker threads for the per-expert passes; set once from --threads. +unsigned g_threads = 1; + +// Runs work(i) for i in [0, n) on g_threads threads and hands each result to +// consume(i, result) in index order, so the output does not depend on the +// thread count. Workers run at most two batches ahead of the consumer. +template +void for_each_expert_ordered(uint32_t n, + const std::function & work, + const std::function & consume) { + const unsigned threads = std::max(1u, std::min(g_threads, n == 0 ? 1 : n)); + const uint32_t window = threads * 2; + std::vector> slots(n); + std::mutex mutex; + std::condition_variable ready; + std::atomic next{0}; + std::exception_ptr error; + uint32_t consumed = 0; + auto worker = [&] { + for (;;) { + const uint32_t i = next.fetch_add(1); + if (i >= n) return; + { + std::unique_lock lock(mutex); + ready.wait(lock, [&] { return error || i < consumed + window; }); + if (error) return; + } + try { + Result result = work(i); + std::lock_guard lock(mutex); + slots[i] = std::move(result); + } catch (...) { + std::lock_guard lock(mutex); + if (!error) error = std::current_exception(); + } + ready.notify_all(); + } + }; + std::vector pool; + for (unsigned t = 0; t < threads; ++t) pool.emplace_back(worker); + for (uint32_t i = 0; i < n; ++i) { + Result result; + { + std::unique_lock lock(mutex); + ready.wait(lock, [&] { return error || slots[i].has_value(); }); + if (error) break; + result = std::move(*slots[i]); + slots[i].reset(); + consumed = i + 1; + } + ready.notify_all(); + consume(i, result); + } + for (auto & t : pool) t.join(); + if (error) std::rethrow_exception(error); +} + +uint64_t checked_mul(uint64_t a, uint64_t b, const std::string & what) { + if (a != 0 && b > std::numeric_limits::max() / a) { + fail("overflow computing " + what); + } + return a*b; +} + +uint64_t element_count(const std::vector & shape, const std::string & name) { + if (shape.empty() || shape.size() > GGML_MAX_DIMS) { + fail("unsupported rank for " + name); + } + uint64_t n = 1; + for (int64_t d : shape) { + if (d <= 0) fail("non-positive dimension for " + name); + n = checked_mul(n, static_cast(d), "element count for " + name); + } + return n; +} + +size_t dtype_size(const std::string & dtype) { + if (dtype == "BF16" || dtype == "F16") return 2; + if (dtype == "F32" || dtype == "I32" || dtype == "U32") return 4; + if (dtype == "I64" || dtype == "U64") return 8; + if (dtype == "I8" || dtype == "U8" || dtype == "F8_E4M3" || dtype == "F8_E8M0") return 1; + fail("unsupported safetensors dtype " + dtype); +} + +uint16_t float_to_bf16(float value) { + uint32_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + const uint32_t rounding = 0x7fffu + ((bits >> 16) & 1u); + return static_cast((bits + rounding) >> 16); +} + +float bf16_to_float(uint16_t value) { + uint32_t bits = static_cast(value) << 16; + float out; + std::memcpy(&out, &bits, sizeof(out)); + return out; +} + +float fp4_e2m1(uint8_t code) { + static constexpr float table[16] = { + 0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, + 0.0f,-0.5f,-1.0f,-1.5f,-2.0f,-3.0f,-4.0f,-6.0f, + }; + return table[code & 15u]; +} + +float fp8_e4m3fn(uint8_t value) { + const int sign = (value & 0x80u) ? -1 : 1; + const int exponent = (value >> 3) & 15; + const int mantissa = value & 7; + if (exponent == 15 && mantissa == 7) { + fail("F8_E4M3 contains NaN encoding"); + } + const float magnitude = exponent == 0 + ? std::ldexp(static_cast(mantissa), -9) + : std::ldexp(static_cast(8 + mantissa), exponent - 10); + return sign * magnitude; +} + +float fp8_e8m0(uint8_t value) { + if (value == 0xffu) fail("F8_E8M0 contains NaN encoding"); + return std::ldexp(1.0f, static_cast(value) - 127); +} + +struct FileDescriptor { + int fd = -1; + explicit FileDescriptor(const fs::path & path) { + fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) fail("open " + path.string() + ": " + std::strerror(errno)); + } + FileDescriptor(const FileDescriptor &) = delete; + FileDescriptor & operator=(const FileDescriptor &) = delete; + ~FileDescriptor() { if (fd >= 0) ::close(fd); } +}; + +void pread_exact(int fd, void * dst, size_t n, uint64_t offset, const std::string & what) { + auto * p = static_cast(dst); + size_t done = 0; + while (done < n) { + const ssize_t got = ::pread(fd, p + done, n - done, static_cast(offset + done)); + if (got < 0 && errno == EINTR) continue; + if (got <= 0) fail("short read for " + what); + done += static_cast(got); + } +} + +struct StEntry { + std::string name; + std::string dtype; + std::vector shape; + fs::path path; + uint64_t offset = 0; + uint64_t size = 0; +}; + +class SafeTensorSet { +public: + explicit SafeTensorSet(fs::path root) : root_(std::move(root)) { + if (!fs::is_directory(root_)) fail("input is not a directory: " + root_.string()); + config_ = read_json(root_ / "config.json"); + tokenizer_ = read_json(root_ / "tokenizer.json"); + const json index = read_json(root_ / "model.safetensors.index.json"); + if (!index.contains("weight_map") || !index["weight_map"].is_object()) { + fail("model.safetensors.index.json has no object weight_map"); + } + std::map weight_map; + std::set shards; + for (const auto & item : index["weight_map"].items()) { + if (!item.value().is_string()) fail("weight_map value is not a string for " + item.key()); + const std::string shard = item.value().get(); + if (fs::path(shard).is_absolute() || shard.find("..") != std::string::npos) { + fail("unsafe shard path " + shard); + } + weight_map.emplace(item.key(), shard); + shards.insert(shard); + } + for (const std::string & shard : shards) load_shard(shard); + if (entries_.size() != weight_map.size()) { + fail("safetensors headers contain " + std::to_string(entries_.size()) + + " tensors but index names " + std::to_string(weight_map.size())); + } + for (const auto & item : weight_map) { + const auto it = entries_.find(item.first); + if (it == entries_.end()) fail("index tensor missing from shard header: " + item.first); + if (it->second.path.filename() != item.second) { + fail("index shard mismatch for " + item.first); + } + } + } + + const StEntry & at(const std::string & name) const { + const auto it = entries_.find(name); + if (it == entries_.end()) fail("missing required tensor " + name); + return it->second; + } + bool contains(const std::string & name) const { return entries_.count(name) != 0; } + const std::unordered_map & entries() const { return entries_; } + const json & config() const { return config_; } + const fs::path & root() const { return root_; } + const json & tokenizer() const { return tokenizer_; } + +private: + static json read_json(const fs::path & path) { + std::ifstream in(path); + if (!in) fail("cannot open " + path.string()); + json result; + try { in >> result; } + catch (const std::exception & e) { fail("invalid JSON " + path.string() + ": " + e.what()); } + return result; + } + + void load_shard(const std::string & filename) { + const fs::path path = root_ / filename; + FileDescriptor fd(path); + struct stat st{}; + if (::fstat(fd.fd, &st) != 0 || st.st_size < 8) fail("invalid shard file " + path.string()); + uint64_t header_len = 0; + pread_exact(fd.fd, &header_len, sizeof(header_len), 0, path.string() + " header length"); +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ + fail("big-endian hosts are not supported"); +#endif + if (header_len == 0 || header_len > static_cast(st.st_size) - 8 || header_len > (1ull << 31)) { + fail("invalid safetensors header length in " + path.string()); + } + std::string header(static_cast(header_len), 0); + pread_exact(fd.fd, header.data(), header.size(), 8, path.string() + " header"); + json parsed; + try { parsed = json::parse(header); } + catch (const std::exception & e) { fail("invalid safetensors header in " + path.string() + ": " + e.what()); } + for (const auto & item : parsed.items()) { + if (item.key() == "__metadata__") continue; + const json & info = item.value(); + if (!info.contains("dtype") || !info.contains("shape") || !info.contains("data_offsets")) { + fail("incomplete safetensors entry " + item.key()); + } + StEntry entry; + entry.name = item.key(); + entry.dtype = info["dtype"].get(); + entry.shape = info["shape"].get>(); + const auto range = info["data_offsets"].get>(); + if (range.size() != 2 || range[1] < range[0]) fail("bad data_offsets for " + entry.name); + entry.path = path; + if (range[0] > std::numeric_limits::max() - 8 - header_len) { + fail("data_offsets overflow for " + entry.name); + } + entry.offset = 8 + header_len + range[0]; + entry.size = range[1] - range[0]; + const uint64_t expected = checked_mul(element_count(entry.shape, entry.name), dtype_size(entry.dtype), "byte size for " + entry.name); + if (entry.size != expected || entry.offset > static_cast(st.st_size) || + entry.size > static_cast(st.st_size) - entry.offset) { + fail("size or bounds mismatch for " + entry.name); + } + if (!entries_.emplace(entry.name, std::move(entry)).second) { + fail("duplicate tensor " + item.key()); + } + } + } + + fs::path root_; + json config_; + json tokenizer_; + std::unordered_map entries_; +}; + +struct ImatrixEntry { + int32_t calls = 0; + std::vector values; +}; +using Imatrix = std::unordered_map; + +int32_t read_i32(std::ifstream & in, const std::string & what) { + int32_t value = 0; + if (!in.read(reinterpret_cast(&value), sizeof(value))) fail("truncated imatrix " + what); + return value; +} + +Imatrix load_imatrix(const fs::path & path) { + std::ifstream in(path, std::ios::binary); + if (!in) fail("cannot open imatrix " + path.string()); + const int32_t count = read_i32(in, "entry count"); + if (count <= 0 || count > 1000000) fail("invalid imatrix entry count"); + Imatrix result; + for (int32_t i = 0; i < count; ++i) { + const int32_t name_len = read_i32(in, "name length"); + if (name_len <= 0 || name_len > 4096) fail("invalid imatrix name length"); + std::string name(static_cast(name_len), 0); + if (!in.read(name.data(), name.size())) fail("truncated imatrix name"); + ImatrixEntry entry; + entry.calls = read_i32(in, "call count"); + const int32_t nvalues = read_i32(in, "value count"); + if (entry.calls <= 0 || nvalues <= 0 || nvalues > (1 << 24)) fail("invalid imatrix entry dimensions for " + name); + entry.values.resize(static_cast(nvalues)); + if (!in.read(reinterpret_cast(entry.values.data()), entry.values.size()*sizeof(float))) { + fail("truncated imatrix values for " + name); + } + for (float value : entry.values) { + if (!std::isfinite(value) || value < 0.0f) fail("invalid imatrix value for " + name); + } + if (!result.emplace(name, std::move(entry)).second) fail("duplicate imatrix entry " + name); + } + char extra = 0; + if (in.read(&extra, 1)) fail("trailing bytes in imatrix " + path.string()); + std::cerr << "[calibration] loaded " << result.size() << " imatrix entries from " << path << "\n"; + return result; +} + +// Importance of each input column of one expert. An entry holds either one +// vector shared by all experts (`in` values) or one per expert (expert-major, +// as llama.cpp's imatrix collects them, at least `experts` of them); per-expert +// is better, since experts see different tokens. +const float * require_imatrix( + const std::optional & imatrix, const std::string & name, size_t in, + uint32_t expert, uint32_t experts) { + if (!imatrix) return nullptr; + const auto it = imatrix->find(name); + if (it == imatrix->end()) fail("imatrix is missing required entry " + name); + const std::vector & values = it->second.values; + if (values.size() == in) return values.data(); + if (values.size() % in == 0 && values.size() / in >= experts) return values.data() + static_cast(expert) * in; + fail("imatrix entry " + name + " has " + std::to_string(values.size()) + " values, expected " + + std::to_string(in) + " or one " + std::to_string(in) + "-wide vector for each of " + + std::to_string(experts) + " experts"); +} + +enum class Surface : uint32_t { Gate = 0, Up = 1, Down = 2 }; +enum class BookSource { GateUpJoint, DownOnly }; + +struct ExpertRecipe { + const char * source_leaf; + const char * target_leaf; + Surface surface; + ggml_type qtype; + BookSource books; + uint32_t levels; +}; + +constexpr std::array kExpertRecipes{{ + {"w1", "ffn_gate_exps.weight", Surface::Gate, GGML_TYPE_Q2_1_ROCMFP2_MIX, BookSource::GateUpJoint, kGuLevels}, + {"w3", "ffn_up_exps.weight", Surface::Up, GGML_TYPE_Q2_1_ROCMFP2_MIX, BookSource::GateUpJoint, kGuLevels}, + {"w2", "ffn_down_exps.weight", Surface::Down, GGML_TYPE_Q3_1_ROCMFP3_MIX, BookSource::DownOnly, kP4Levels}, +}}; + +// The shipped DeepSeek-V4-Flash recipe keeps fp2 for the down experts of a +// few layers and fp3 elsewhere. --down-fp2-layers sets the fp2 layers. +constexpr ExpertRecipe kDownFp2{"w2", "ffn_down_exps.weight", Surface::Down, GGML_TYPE_Q2_1_ROCMFP2_MIX, + BookSource::DownOnly, kGuLevels}; +constexpr const char * kDefaultDownFp2Layers = "0,2-4,6,10,11,17-20,39-42"; +std::set g_fp2_down_layers; + +const ExpertRecipe & down_recipe(int layer) { + return g_fp2_down_layers.count(layer) ? kDownFp2 : kExpertRecipes[2]; +} + +std::array layer_recipes(int layer) { + return {&kExpertRecipes[0], &kExpertRecipes[1], &down_recipe(layer)}; +} + +// "0,2-4,39-42" -> {0, 2, 3, 4, 39, 40, 41, 42} +std::set parse_layer_set(const std::string & text) { + std::set out; + std::string item; + std::stringstream in(text); + while (std::getline(in, item, ',')) { + if (item.empty()) continue; + const auto dash = item.find('-'); + const int lo = std::stoi(item.substr(0, dash)); + const int hi = dash == std::string::npos ? lo : std::stoi(item.substr(dash + 1)); + if (lo < 0 || hi < lo) fail("bad layer range: " + item); + for (int layer = lo; layer <= hi; ++layer) out.insert(layer); + } + return out; +} + +std::string source_expert_name(int layer, int expert, const ExpertRecipe & recipe, const char * suffix) { + return "layers." + std::to_string(layer) + ".ffn.experts." + std::to_string(expert) + + "." + recipe.source_leaf + "." + suffix; +} +std::string target_expert_name(int layer, const ExpertRecipe & recipe) { + return "blk." + std::to_string(layer) + "." + recipe.target_leaf; +} + +struct TensorShape { + uint32_t in = 0; + uint32_t out = 0; +}; + +TensorShape validate_expert_source( + const SafeTensorSet & source, int layer, int expert, const ExpertRecipe & recipe) { + const StEntry & weight = source.at(source_expert_name(layer, expert, recipe, "weight")); + const StEntry & scale = source.at(source_expert_name(layer, expert, recipe, "scale")); + if (weight.dtype != "I8" || weight.shape.size() != 2) { + fail("expert weight must be rank-2 packed I8: " + weight.name); + } + const int64_t out = weight.shape[0]; + const int64_t in = weight.shape[1]*2; + if (out <= 0 || in <= 0 || in % kBlock != 0 || out > UINT32_MAX || in > UINT32_MAX) { + fail("unsupported expert dimensions for " + weight.name); + } + if (recipe.qtype == GGML_TYPE_Q2_1_ROCMFP2_MIX && in % 128 != 0) { + fail("qtype-106 input dimension must be a multiple of 128: " + weight.name); + } + if (scale.dtype != "F8_E8M0" || scale.shape != std::vector{out, in/32}) { + fail("expert scale shape/dtype mismatch for " + scale.name); + } + return {static_cast(in), static_cast(out)}; +} + +class HistogramFitter { +public: + static constexpr int kBins = 4097; + HistogramFitter() { for (auto & h : hist_) h.assign(kBins, 0.0); } + + void add_half(const float values[16], const float * importance) { + float max_abs = 0.0f; + double sum_sq = 0.0; + for (int i = 0; i < 16; ++i) { + if (!std::isfinite(values[i])) fail("non-finite expert value during calibration"); + max_abs = std::max(max_abs, std::fabs(values[i])); + sum_sq += static_cast(values[i])*values[i]; + } + if (!(max_abs > 0.0f)) { + hist_[0][kBins/2] += 16.0; + return; + } + const double rms_ratio = std::sqrt(sum_sq/16.0)/max_abs; + const int population = rms_ratio < 0.52 ? 0 : 1; + for (int i = 0; i < 16; ++i) { + const float normalized = std::max(-1.0f, std::min(1.0f, values[i]/max_abs)); + const int bin = std::max(0, std::min(kBins - 1, + static_cast(std::lround((normalized + 1.0f)*0.5f*(kBins - 1))))); + double w = importance ? importance[i] : 1.0; + if (w > 0.0 && std::isfinite(w)) hist_[population][bin] += w; + } + } + + static std::vector round_centers(const std::vector & centers, bool & repaired) { + std::vector bits; + for (float center : centers) { + if (!std::isfinite(center) || center < -1.0f || center > 1.0f) + fail("invalid adaptive codebook center"); + bits.push_back(float_to_bf16(center)); + } + for (size_t i = 1; i < bits.size(); ++i) { + if (bf16_to_float(bits[i]) <= bf16_to_float(bits[i-1])) { + bits[i] = bf16_step(bits[i-1], true); + repaired = true; + } + } + if (!bits.empty() && bf16_to_float(bits.back()) > 1.0f) { + bits.back() = float_to_bf16(1.0f); + for (size_t i = bits.size()-1; i > 0; --i) { + if (bf16_to_float(bits[i-1]) >= bf16_to_float(bits[i])) + bits[i-1] = bf16_step(bits[i], false); + } + } + float previous = -std::numeric_limits::infinity(); + for (uint16_t b : bits) { + const float value = bf16_to_float(b); + if (!std::isfinite(value) || value < -1.0f || value > 1.0f || value <= previous) + fail("cannot separate adaptive codebook levels in BF16"); + previous = value; + } + return bits; + } + + std::vector fit(int levels, const std::string & label = "unlabelled", + std::vector * repairs = nullptr) const { + if (levels != 4 && levels != 8) fail("unsupported fitted codebook size"); + std::vector result; + result.reserve(static_cast(2*levels)); + std::vector fallback(kBins, 0.0); + for (int i = 0; i < kBins; ++i) fallback[i] = hist_[0][i] + hist_[1][i]; + for (int population = 0; population < 2; ++population) { + const std::vector & h = total(hist_[population]) > 0.0 ? hist_[population] : fallback; + const std::vector centers = lloyd(h, levels); + bool repaired = false; + const auto bits = round_centers(centers, repaired); + result.insert(result.end(), bits.begin(), bits.end()); + if (repaired) { + const std::string stamp = label + " population=" + std::to_string(population) + + " levels=" + std::to_string(levels) + " repair=bf16-epsilon-v1"; + std::cerr << "[calibration] WARNING: " << stamp << "\n"; + if (repairs) repairs->push_back(stamp); + } + } + return result; + } + +private: + static uint16_t bf16_step(uint16_t bits, bool up) { + if ((bits & 0x7fffu) == 0) return up ? 0x0080u : 0x8080u; + return static_cast(bits + ((up != ((bits & 0x8000u) != 0)) ? 1 : -1)); + } + static double total(const std::vector & h) { + double out = 0.0; + for (double value : h) out += value; + return out; + } + static float coordinate(int bin) { + return -1.0f + 2.0f*static_cast(bin)/static_cast(kBins - 1); + } + static std::vector lloyd(const std::vector & h, int k) { + const double mass = total(h); + if (!(mass > 0.0)) fail("cannot fit an empty codebook histogram"); + std::vector c(static_cast(k)); + for (int j = 0; j < k; ++j) { + const double target = mass*(j + 0.5)/k; + double running = 0.0; + int bin = 0; + for (; bin < kBins - 1; ++bin) { + running += h[bin]; + if (running >= target) break; + } + c[j] = coordinate(bin); + } + for (int iteration = 0; iteration < 24; ++iteration) { + std::vector sums(static_cast(k), 0.0); + std::vector weights(static_cast(k), 0.0); + int cluster = 0; + for (int bin = 0; bin < kBins; ++bin) { + const float x = coordinate(bin); + while (cluster + 1 < k && x > 0.5f*(c[cluster] + c[cluster + 1])) ++cluster; + sums[cluster] += h[bin]*x; + weights[cluster] += h[bin]; + } + for (int j = 0; j < k; ++j) if (weights[j] > 0.0) c[j] = static_cast(sums[j]/weights[j]); + std::sort(c.begin(), c.end()); + } + return c; + } + + std::array, 2> hist_; +}; + +struct OpenTensorPair { + FileDescriptor weight_fd; + FileDescriptor scale_fd; + const StEntry & weight; + const StEntry & scale; + OpenTensorPair(const StEntry & w, const StEntry & s) + : weight_fd(w.path), scale_fd(s.path), weight(w), scale(s) {} +}; + +void decode_expert_row( + const OpenTensorPair & input, uint32_t row, uint32_t in, + std::vector & packed, std::vector & scales, + std::vector & values) { + packed.resize(in/2); + scales.resize(in/32); + values.resize(in); + pread_exact(input.weight_fd.fd, packed.data(), packed.size(), + input.weight.offset + static_cast(row)*packed.size(), input.weight.name); + pread_exact(input.scale_fd.fd, scales.data(), scales.size(), + input.scale.offset + static_cast(row)*scales.size(), input.scale.name); + for (uint32_t col = 0; col < in; ++col) { + const uint8_t byte = packed[col/2]; + const uint8_t nibble = (col & 1) ? (byte >> 4) : (byte & 15u); + values[col] = fp4_e2m1(nibble)*fp8_e8m0(scales[col/32]); + if (!std::isfinite(values[col])) fail("non-finite decoded expert value in " + input.weight.name); + } +} + +void add_expert_to_fitter( + const SafeTensorSet & source, int layer, int expert, + const ExpertRecipe & recipe, const float * importance, + HistogramFitter & fitter) { + const TensorShape shape = validate_expert_source(source, layer, expert, recipe); + const StEntry & w = source.at(source_expert_name(layer, expert, recipe, "weight")); + const StEntry & s = source.at(source_expert_name(layer, expert, recipe, "scale")); + OpenTensorPair input(w, s); + std::vector packed, scales; + std::vector values; + for (uint32_t row = 0; row < shape.out; ++row) { + decode_expert_row(input, row, shape.in, packed, scales, values); + for (uint32_t col = 0; col < shape.in; col += 16) { + fitter.add_half(values.data() + col, + importance ? importance + col : nullptr); + } + } +} + +struct CodebookRegistry { + uint32_t levels = 0; + std::vector> experts; +}; + +struct LayerCalibration { + int layer = 0; + TensorShape gate_up_shape; + TensorShape down_shape; + CodebookRegistry gate_up; + CodebookRegistry down; + std::vector repairs; +}; + +struct Options { + fs::path input; + fs::path output; + std::optional imatrix; + bool absmax_only = false; + bool experts_only = false; + bool force = false; + bool validate_input_only = false; + int layer_count = -1; + int expert_limit = -1; + int threads = 0; // 0 = every core + std::string down_fp2_layers = kDefaultDownFp2Layers; +}; + +void usage(const char * argv0) { + std::cerr << "Usage: " << argv0 << " --input DIR --output FILE (--imatrix FILE | --absmax-only)\n" + << " [--layer-count N] [--expert-limit N] [--experts-only] [--validate-input-only] [--force] [--threads N]\n" + << " [--down-fp2-layers LIST] fp2 instead of fp3 for the down experts of these layers (default " + << kDefaultDownFp2Layers << ")\n"; +} + +int parse_nonnegative(const char * value, const std::string & option, bool allow_zero = true) { + char * end = nullptr; + errno = 0; + const long parsed = std::strtol(value, &end, 10); + if (errno || !end || *end || parsed < (allow_zero ? 0 : 1) || parsed > INT32_MAX) { + fail("invalid value for " + option + ": " + value); + } + return static_cast(parsed); +} + +Options parse_options(int argc, char ** argv) { + Options out; + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + auto value = [&]() -> const char * { + if (++i >= argc) fail("missing value after " + arg); + return argv[i]; + }; + if (arg == "--input") out.input = value(); + else if (arg == "--output") out.output = value(); + else if (arg == "--imatrix") out.imatrix = fs::path(value()); + else if (arg == "--absmax-only") out.absmax_only = true; + else if (arg == "--experts-only") out.experts_only = true; + else if (arg == "--force") out.force = true; + else if (arg == "--validate-input-only") out.validate_input_only = true; + else if (arg == "--layer-count") out.layer_count = parse_nonnegative(value(), arg, false); + else if (arg == "--expert-limit") out.expert_limit = parse_nonnegative(value(), arg, false); + else if (arg == "--threads") out.threads = parse_nonnegative(value(), arg, false); + else if (arg == "--down-fp2-layers") out.down_fp2_layers = value(); + else if (arg == "--help" || arg == "-h") { usage(argv[0]); std::exit(0); } + else fail("unknown option " + arg); + } + if (out.input.empty() || out.output.empty()) fail("--input and --output are required"); + if (out.absmax_only == out.imatrix.has_value()) { + fail("choose exactly one of --imatrix FILE or --absmax-only"); + } + return out; +} + +uint32_t config_u32(const json & c, const char * key, uint32_t fallback = 0) { + if (!c.contains(key)) { + if (fallback != 0) return fallback; + fail(std::string("config.json is missing ") + key); + } + const int64_t value = c[key].get(); + if (value <= 0 || value > UINT32_MAX) fail(std::string("invalid config value ") + key); + return static_cast(value); +} + +float config_f32(const json & c, const char * key, float fallback) { + if (!c.contains(key)) return fallback; + const float value = c[key].get(); + if (!std::isfinite(value)) fail(std::string("non-finite config value ") + key); + return value; +} + +template +void append_le(std::vector & out, T value) { +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ + fail("big-endian hosts are not supported"); +#endif + const auto * p = reinterpret_cast(&value); + out.insert(out.end(), p, p + sizeof(value)); +} + +std::vector make_p4_blob(const std::vector & layers, uint32_t experts) { + std::vector out; + std::vector fp3; + for (const LayerCalibration & layer : layers) { + if (layer.down.levels == kP4Levels) fp3.push_back(&layer); + } + if (fp3.empty()) return out; + const char magic[8] = {'P','4','M','I','X','v','1','\0'}; + out.insert(out.end(), magic, magic + 8); + append_le(out, static_cast(fp3.size())); + append_le(out, 0); + for (const LayerCalibration * layer_ptr : fp3) { + const LayerCalibration & layer = *layer_ptr; + if (layer.down.experts.size() != experts) { + fail("incomplete qtype-105 codebook registry at layer " + std::to_string(layer.layer)); + } + append_le(out, static_cast(layer.layer)); + append_le(out, experts); + append_le(out, layer.down_shape.out); + append_le(out, layer.down_shape.in); + append_le(out, kCodebooks); + append_le(out, kP4Levels); + out.insert(out.end(), experts, 1u); + out.insert(out.end(), experts, 0u); + for (const auto & book : layer.down.experts) { + if (book.size() != kCodebooks*kP4Levels) fail("bad qtype-105 codebook length"); + for (uint16_t value : book) append_le(out, value); + } + } + return out; +} + +std::vector make_gumix_blob(const std::vector & layers, uint32_t experts) { + std::vector out; + const char magic[8] = {'G','U','M','I','X','s','1','\0'}; + out.insert(out.end(), magic, magic + 8); + uint32_t entries = 0; + for (const LayerCalibration & layer : layers) entries += layer.down.levels == kGuLevels ? 3 : 2; + append_le(out, entries); + append_le(out, 0); + for (const LayerCalibration & layer : layers) { + if (layer.gate_up.levels != kGuLevels || layer.gate_up.experts.size() != experts) { + fail("incomplete qtype-106 codebook registry at layer " + std::to_string(layer.layer)); + } + for (Surface surface : {Surface::Gate, Surface::Up}) { + append_le(out, static_cast(layer.layer)); + append_le(out, static_cast(surface)); + append_le(out, experts); + append_le(out, layer.gate_up_shape.out); + append_le(out, layer.gate_up_shape.in); + append_le(out, kCodebooks); + append_le(out, kGuLevels); + out.insert(out.end(), experts, 1u); + for (const auto & book : layer.gate_up.experts) { + if (book.size() != kCodebooks*kGuLevels) fail("bad qtype-106 codebook length"); + for (uint16_t value : book) append_le(out, value); + } + } + if (layer.down.levels != kGuLevels) continue; + if (layer.down.experts.size() != experts) { + fail("incomplete qtype-106 down codebook registry at layer " + std::to_string(layer.layer)); + } + append_le(out, static_cast(layer.layer)); + append_le(out, static_cast(Surface::Down)); + append_le(out, experts); + append_le(out, layer.down_shape.out); + append_le(out, layer.down_shape.in); + append_le(out, kCodebooks); + append_le(out, kGuLevels); + out.insert(out.end(), experts, 1u); + for (const auto & book : layer.down.experts) { + if (book.size() != kCodebooks*kGuLevels) fail("bad qtype-106 down codebook length"); + for (uint16_t value : book) append_le(out, value); + } + } + return out; +} + +enum class Producer { Raw, Dense, Int64ToInt32, Expert }; + +struct TensorSpec { + std::string name; + ggml_type type = GGML_TYPE_COUNT; + std::vector ne; + Producer producer = Producer::Raw; + const StEntry * source = nullptr; + const StEntry * scale = nullptr; + int layer = -1; + const ExpertRecipe * recipe = nullptr; +}; + +std::vector reverse_shape(const StEntry & entry) { + std::vector ne(entry.shape.rbegin(), entry.shape.rend()); + while (ne.size() < GGML_MAX_DIMS) ne.push_back(1); + return ne; +} + +ggml_type direct_ggml_type(const std::string & dtype) { + if (dtype == "BF16") return GGML_TYPE_BF16; + if (dtype == "F16") return GGML_TYPE_F16; + if (dtype == "F32") return GGML_TYPE_F32; + if (dtype == "I32") return GGML_TYPE_I32; + fail("unsupported direct tensor dtype " + dtype); +} + +const std::unordered_map kLayerNameMap = { + {"attn.attn_sink", "attn_sinks.weight"}, + {"attn.kv_norm.weight", "attn_kv_a_norm.weight"}, + {"attn.q_norm.weight", "attn_q_a_norm.weight"}, + {"attn.wkv.weight", "attn_kv.weight"}, + {"attn.wo_a.weight", "attn_output_a.weight"}, + {"attn.wo_b.weight", "attn_output_b.weight"}, + {"attn.wq_a.weight", "attn_q_a.weight"}, + {"attn.wq_b.weight", "attn_q_b.weight"}, + {"attn_norm.weight", "attn_norm.weight"}, + {"attn.compressor.ape", "attn_compressor_ape.weight"}, + {"attn.compressor.norm.weight", "attn_compressor_norm.weight"}, + {"attn.compressor.wgate.weight", "attn_compressor_gate.weight"}, + {"attn.compressor.wkv.weight", "attn_compressor_kv.weight"}, + {"attn.indexer.compressor.ape", "indexer_compressor_ape.weight"}, + {"attn.indexer.compressor.norm.weight", "indexer_compressor_norm.weight"}, + {"attn.indexer.compressor.wgate.weight", "indexer_compressor_gate.weight"}, + {"attn.indexer.compressor.wkv.weight", "indexer_compressor_kv.weight"}, + {"attn.indexer.weights_proj.weight", "indexer.proj.weight"}, + {"attn.indexer.wq_b.weight", "indexer.attn_q_b.weight"}, + {"ffn.gate.bias", "exp_probs_b.bias"}, + {"ffn.gate.tid2eid", "ffn_gate_tid2eid.weight"}, + {"ffn.gate.weight", "ffn_gate_inp.weight"}, + {"ffn.shared_experts.w1.weight", "ffn_gate_shexp.weight"}, + {"ffn.shared_experts.w2.weight", "ffn_down_shexp.weight"}, + {"ffn.shared_experts.w3.weight", "ffn_up_shexp.weight"}, + {"ffn_norm.weight", "ffn_norm.weight"}, + {"hc_attn_base", "hc_attn_base.weight"}, + {"hc_attn_fn", "hc_attn_fn.weight"}, + {"hc_attn_scale", "hc_attn_scale.weight"}, + {"hc_ffn_base", "hc_ffn_base.weight"}, + {"hc_ffn_fn", "hc_ffn_fn.weight"}, + {"hc_ffn_scale", "hc_ffn_scale.weight"}, +}; + +const std::unordered_map kGlobalNameMap = { + {"embed.weight", "token_embd.weight"}, + {"head.weight", "output.weight"}, + {"norm.weight", "output_norm.weight"}, + {"hc_head_base", "output_hc_base.weight"}, + {"hc_head_fn", "output_hc_fn.weight"}, + {"hc_head_scale", "output_hc_scale.weight"}, +}; + +bool starts_with(const std::string & value, const std::string & prefix) { + return value.size() >= prefix.size() && value.compare(0, prefix.size(), prefix) == 0; +} +bool ends_with(const std::string & value, const std::string & suffix) { + return value.size() >= suffix.size() && value.compare(value.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +std::optional> parse_layer_name(const std::string & name) { + if (!starts_with(name, "layers.")) return std::nullopt; + const size_t dot = name.find('.', 7); + if (dot == std::string::npos) fail("malformed layer tensor name " + name); + const std::string number = name.substr(7, dot - 7); + if (number.empty() || number.find_first_not_of("0123456789") != std::string::npos) { + fail("malformed layer index in " + name); + } + return std::make_pair(std::stoi(number), name.substr(dot + 1)); +} + +TensorSpec mapped_source_spec(const std::string & target, const StEntry & source, + const SafeTensorSet & all) { + TensorSpec spec; + spec.name = target; + spec.source = &source; + if (source.dtype == "I64") { + if (!ends_with(source.name, ".tid2eid")) fail("unsupported I64 tensor " + source.name); + spec.type = GGML_TYPE_I32; + spec.producer = Producer::Int64ToInt32; + } else if (source.dtype == "F8_E4M3") { + if (!ends_with(source.name, ".weight")) fail("FP8 tensor is not a weight: " + source.name); + const std::string scale_name = source.name.substr(0, source.name.size() - 6) + "scale"; + const StEntry & scale = all.at(scale_name); + if (scale.dtype != "F8_E8M0" || source.shape.size() != 2 || scale.shape.size() != 2) { + fail("unsupported FP8 tensor/scale pair " + source.name); + } + const std::vector expected = { + (source.shape[0] + 127)/128, (source.shape[1] + 127)/128, + }; + if (scale.shape != expected) fail("FP8 scale shape mismatch for " + source.name); + spec.type = GGML_TYPE_Q4_0_ROCMFP4_FAST; + spec.producer = Producer::Dense; + spec.scale = &scale; + } else if (source.dtype == "BF16" && source.shape.size() == 2 && + (target == "token_embd.weight" || target == "output.weight")) { + spec.type = target == "token_embd.weight" ? GGML_TYPE_Q6_K : GGML_TYPE_Q4_0_ROCMFP4_FAST; + spec.producer = Producer::Dense; + } else { + spec.type = direct_ggml_type(source.dtype); + spec.producer = Producer::Raw; + } + if (spec.producer == Producer::Dense) { + const int64_t block = ggml_blck_size(spec.type); + if (source.shape[1] % block != 0) { + fail("row of " + source.name + " is not a multiple of " + std::to_string(block)); + } + } + spec.ne = reverse_shape(source); + if (spec.name.size() >= GGML_MAX_NAME) fail("GGUF tensor name too long: " + spec.name); + return spec; +} + +std::vector make_plan( + const SafeTensorSet & source, const std::vector & layers, + uint32_t experts, bool experts_only) { + std::vector plan; + std::set names; + auto add = [&](TensorSpec spec) { + if (!names.insert(spec.name).second) fail("duplicate output tensor name " + spec.name); + plan.push_back(std::move(spec)); + }; + + if (!experts_only) { + for (const auto & mapping : kGlobalNameMap) { + add(mapped_source_spec(mapping.second, source.at(mapping.first), source)); + } + } + + for (const LayerCalibration & layer : layers) { + if (!experts_only) { + const std::string prefix = "layers." + std::to_string(layer.layer) + "."; + for (const auto & mapping : kLayerNameMap) { + const std::string src = prefix + mapping.first; + if (!source.contains(src)) continue; + if (ends_with(mapping.first, ".scale")) continue; + add(mapped_source_spec("blk." + std::to_string(layer.layer) + "." + mapping.second, + source.at(src), source)); + } + } + for (const ExpertRecipe * recipe_ptr : layer_recipes(layer.layer)) { + const ExpertRecipe & recipe = *recipe_ptr; + const TensorShape shape = recipe.books == BookSource::GateUpJoint + ? layer.gate_up_shape : layer.down_shape; + TensorSpec spec; + spec.name = target_expert_name(layer.layer, recipe); + spec.type = recipe.qtype; + spec.ne = {shape.in, shape.out, static_cast(experts), 1}; + spec.producer = Producer::Expert; + spec.layer = layer.layer; + spec.recipe = &recipe; + add(std::move(spec)); + } + } + + if (!experts_only) { + bool dropped_mtp = false; + static const std::regex expert_leaf( + R"(^ffn\.experts\.[0-9]+\.(w1|w2|w3)\.(weight|scale)$)"); + for (const auto & item : source.entries()) { + const std::string & name = item.first; + if (starts_with(name, "vision.") || starts_with(name, "aligner.") || starts_with(name, "image_")) { + TensorSpec spec = mapped_source_spec(name, item.second, source); + if (spec.producer != Producer::Raw) fail("vision pass-through is not byte-lossless for " + name); + add(std::move(spec)); + continue; + } + if (starts_with(name, "mtp.")) { + dropped_mtp = true; + continue; + } + if (kGlobalNameMap.count(name)) continue; + const auto parsed = parse_layer_name(name); + if (!parsed) fail("unsupported top-level tensor " + name); + const int layer = parsed->first; + const std::string & leaf = parsed->second; + const bool selected = std::any_of(layers.begin(), layers.end(), + [layer](const LayerCalibration & c) { return c.layer == layer; }); + if (!selected) continue; + if (leaf == "ffn.gate.bias_vl") { + TensorSpec spec = mapped_source_spec(name, item.second, source); + if (spec.producer != Producer::Raw) fail("bias_vl pass-through is not byte-lossless for " + name); + add(std::move(spec)); + continue; + } + if (starts_with(leaf, "ffn.experts.")) { + if (!std::regex_match(leaf, expert_leaf)) fail("unsupported expert tensor " + name); + continue; + } + if (kLayerNameMap.count(leaf) || (ends_with(leaf, ".scale") && + source.contains(name.substr(0, name.size() - 5) + "weight"))) continue; + fail("unsupported selected-layer tensor " + name); + } + if (dropped_mtp) { + std::cerr << "[plan] NOTE: mtp.* predictor tensors are intentionally omitted; " + << "the DeepSeek4 target loader does not consume them\n"; + } + } + + std::sort(plan.begin(), plan.end(), [](const TensorSpec & a, const TensorSpec & b) { + return a.name < b.name; + }); + return plan; +} + +// The checkpoint's own name when its config carries one, else the input +// directory's name; either way marked as a MIX build. +std::string model_name(const SafeTensorSet & source) { + const json & c = source.config(); + std::string base; + if (c.contains("_name_or_path") && c["_name_or_path"].is_string()) base = c["_name_or_path"].get(); + if (base.empty()) base = source.root().filename().string(); + const auto slash = base.find_last_of('/'); + if (slash != std::string::npos) base = base.substr(slash + 1); + return base.empty() ? "DeepSeek-V4 MIX" : base + " MIX"; +} + +void set_model_metadata(gguf_context * ctx, const SafeTensorSet & source, + uint32_t layers, uint32_t experts, bool absmax_only, + bool smoke_artifact, const std::vector & p4_blob, + const std::vector & gumix_blob, const std::string & calibration) { + const json & c = source.config(); + gguf_set_val_str(ctx, "general.architecture", "deepseek4"); + gguf_set_val_str(ctx, "general.name", model_name(source).c_str()); + gguf_set_val_u32(ctx, "general.alignment", kAlignment); + gguf_set_val_u32(ctx, "general.file_type", 119); + gguf_set_val_str(ctx, "deepseek4.mix.calibration", + absmax_only ? "absmax-only (LOWER QUALITY; no imatrix)" : calibration.c_str()); + gguf_set_val_bool(ctx, "deepseek4.mix.lower_quality_absmax_only", absmax_only); + gguf_set_val_bool(ctx, "deepseek4.mix.experts_only_smoke_artifact", smoke_artifact); + + gguf_set_val_u32(ctx, "deepseek4.block_count", layers); + gguf_set_val_u32(ctx, "deepseek4.embedding_length", config_u32(c, "hidden_size")); + gguf_set_val_u32(ctx, "deepseek4.vocab_size", config_u32(c, "vocab_size")); + gguf_set_val_u32(ctx, "deepseek4.attention.head_count", config_u32(c, "num_attention_heads")); + gguf_set_val_u32(ctx, "deepseek4.attention.head_count_kv", config_u32(c, "num_key_value_heads")); + gguf_set_val_u32(ctx, "deepseek4.attention.key_length", config_u32(c, "head_dim")); + gguf_set_val_u32(ctx, "deepseek4.rope.dimension_count", config_u32(c, "qk_rope_head_dim")); + gguf_set_val_u32(ctx, "deepseek4.attention.q_lora_rank", config_u32(c, "q_lora_rank")); + gguf_set_val_u32(ctx, "deepseek4.attention.output_lora_rank", config_u32(c, "o_lora_rank")); + gguf_set_val_u32(ctx, "deepseek4.attention.output_group_count", config_u32(c, "o_groups")); + gguf_set_val_u32(ctx, "deepseek4.expert_count", experts); + gguf_set_val_u32(ctx, "deepseek4.expert_used_count", std::min(experts, config_u32(c, "num_experts_per_tok"))); + gguf_set_val_u32(ctx, "deepseek4.expert_shared_count", config_u32(c, "n_shared_experts")); + gguf_set_val_u32(ctx, "deepseek4.expert_feed_forward_length", config_u32(c, "moe_intermediate_size")); + gguf_set_val_u32(ctx, "deepseek4.hash_layer_count", std::min(layers, config_u32(c, "num_hash_layers"))); + gguf_set_val_u32(ctx, "deepseek4.attention.sliding_window", config_u32(c, "sliding_window")); + gguf_set_val_u32(ctx, "deepseek4.attention.indexer.head_count", config_u32(c, "index_n_heads")); + gguf_set_val_u32(ctx, "deepseek4.attention.indexer.key_length", config_u32(c, "index_head_dim")); + gguf_set_val_u32(ctx, "deepseek4.attention.indexer.top_k", config_u32(c, "index_topk")); + gguf_set_val_u32(ctx, "deepseek4.hyper_connection.count", config_u32(c, "hc_mult")); + gguf_set_val_u32(ctx, "deepseek4.hyper_connection.sinkhorn_iterations", config_u32(c, "hc_sinkhorn_iters")); + + gguf_set_val_f32(ctx, "deepseek4.rope.freq_base", config_f32(c, "rope_theta", 10000.0f)); + gguf_set_val_f32(ctx, "deepseek4.rope.scaling.factor", 16.0f); + gguf_set_val_f32(ctx, "deepseek4.rope.scaling.yarn_beta_fast", 32.0f); + gguf_set_val_f32(ctx, "deepseek4.rope.scaling.yarn_beta_slow", 1.0f); + gguf_set_val_f32(ctx, "deepseek4.attention.compress_rope_freq_base", config_f32(c, "compress_rope_theta", 160000.0f)); + gguf_set_val_u64(ctx, "deepseek4.rope.scaling.original_context_length", 65536); + gguf_set_val_f32(ctx, "deepseek4.attention.layer_norm_rms_epsilon", 1e-6f); + gguf_set_val_f32(ctx, "deepseek4.hyper_connection.epsilon", config_f32(c, "hc_eps", 1e-6f)); + gguf_set_val_f32(ctx, "deepseek4.expert_weights_scale", config_f32(c, "routed_scaling_factor", 1.5f)); + gguf_set_val_f32(ctx, "deepseek4.swiglu_clamp_exp", config_f32(c, "swiglu_limit", 10.0f)); + + // An all-zero schedule would load and silently run the wrong attention. + if (!c.contains("compress_ratios") || !c["compress_ratios"].is_array()) fail("config.json has no compress_ratios array"); + if (c["compress_ratios"].size() < layers) fail("config compress_ratios is shorter than selected layers"); + std::vector ratios(layers, 0); + for (uint32_t i = 0; i < layers; ++i) ratios[i] = c["compress_ratios"][i].get(); + gguf_set_arr_data(ctx, "deepseek4.attention.compress_ratios", GGUF_TYPE_UINT32, + ratios.data(), ratios.size()); + + gguf_set_val_str(ctx, "tokenizer.ggml.model", "gpt2"); + gguf_set_val_str(ctx, "tokenizer.ggml.pre", "deepseek-v3"); + gguf_set_val_u32(ctx, "tokenizer.ggml.bos_token_id", c.value("bos_token_id", 0u)); + gguf_set_val_u32(ctx, "tokenizer.ggml.eos_token_id", c.value("eos_token_id", 1u)); + gguf_set_val_u32(ctx, "tokenizer.ggml.padding_token_id", c.value("eos_token_id", 1u)); + gguf_set_val_bool(ctx, "tokenizer.ggml.add_bos_token", false); + gguf_set_val_bool(ctx, "tokenizer.ggml.add_eos_token", false); + + const uint32_t vocab_size = config_u32(c, "vocab_size"); + std::vector tokens(vocab_size); + std::vector types(vocab_size, 1); + std::vector scores(vocab_size, 0.0f); + std::vector assigned(vocab_size, false); + const json & tok = source.tokenizer(); + if (!tok.contains("model") || !tok["model"].contains("vocab") || + !tok["model"].contains("merges")) fail("tokenizer.json is missing BPE model fields"); + for (const auto & item : tok["model"]["vocab"].items()) { + const int64_t id = item.value().get(); + if (id < 0 || id >= vocab_size) fail("tokenizer vocab id out of range"); + tokens[id] = item.key(); + assigned[id] = true; + } + if (tok.contains("added_tokens")) { + for (const json & added : tok["added_tokens"]) { + const int64_t id = added.at("id").get(); + if (id < 0 || id >= vocab_size) fail("added token id out of range"); + tokens[id] = added.at("content").get(); + types[id] = added.value("special", true) ? 3 : 4; + assigned[id] = true; + } + } + for (uint32_t i = 0; i < vocab_size; ++i) { + if (!assigned[i]) fail("tokenizer has no token for id " + std::to_string(i)); + } + std::vector token_ptrs; + token_ptrs.reserve(tokens.size()); + for (const std::string & token : tokens) token_ptrs.push_back(token.c_str()); + gguf_set_arr_str(ctx, "tokenizer.ggml.tokens", token_ptrs.data(), token_ptrs.size()); + gguf_set_arr_data(ctx, "tokenizer.ggml.scores", GGUF_TYPE_FLOAT32, scores.data(), scores.size()); + gguf_set_arr_data(ctx, "tokenizer.ggml.token_type", GGUF_TYPE_INT32, types.data(), types.size()); + + std::vector merges = tok["model"]["merges"].get>(); + std::vector merge_ptrs; + merge_ptrs.reserve(merges.size()); + for (const std::string & merge : merges) merge_ptrs.push_back(merge.c_str()); + gguf_set_arr_str(ctx, "tokenizer.ggml.merges", merge_ptrs.data(), merge_ptrs.size()); + + if (!p4_blob.empty()) { + gguf_set_arr_data(ctx, "deepseek4.p4mix.sidecar", GGUF_TYPE_UINT8, + p4_blob.data(), p4_blob.size()); + } + gguf_set_arr_data(ctx, "deepseek4.gumix.sidecar", GGUF_TYPE_UINT8, + gumix_blob.data(), gumix_blob.size()); +} + +std::unique_ptr make_tensor_descriptor(const TensorSpec & spec) { + auto tensor = std::make_unique(); + std::memset(tensor.get(), 0, sizeof(*tensor)); + tensor->type = spec.type; + for (int i = 0; i < GGML_MAX_DIMS; ++i) tensor->ne[i] = i < static_cast(spec.ne.size()) ? spec.ne[i] : 1; + const int64_t block = ggml_blck_size(spec.type); + if (block <= 0 || tensor->ne[0] % block != 0) { + fail("tensor " + spec.name + " ne[0] violates qtype block size"); + } + tensor->nb[0] = ggml_type_size(spec.type); + tensor->nb[1] = tensor->nb[0]*static_cast(tensor->ne[0]/block); + for (int i = 2; i < GGML_MAX_DIMS; ++i) tensor->nb[i] = tensor->nb[i - 1]*static_cast(tensor->ne[i - 1]); + std::snprintf(tensor->name, sizeof(tensor->name), "%s", spec.name.c_str()); + return tensor; +} + +void fwrite_exact(FILE * out, const void * data, size_t n, const std::string & what) { + if (n && std::fwrite(data, 1, n, out) != n) fail("failed writing " + what); +} + +void copy_raw(FILE * out, const StEntry & source) { + FileDescriptor fd(source.path); + std::vector buffer(std::min(source.size, kCopyChunk)); + uint64_t done = 0; + while (done < source.size) { + const size_t n = static_cast(std::min(buffer.size(), source.size - done)); + pread_exact(fd.fd, buffer.data(), n, source.offset + done, source.name); + fwrite_exact(out, buffer.data(), n, source.name); + done += n; + } +} + +void write_int64_to_int32(FILE * out, const StEntry & source) { + FileDescriptor fd(source.path); + const uint64_t count = element_count(source.shape, source.name); + std::vector input(std::min(count, kCopyChunk/sizeof(int64_t))); + std::vector output(input.size()); + uint64_t done = 0; + while (done < count) { + const size_t n = static_cast(std::min(input.size(), count - done)); + pread_exact(fd.fd, input.data(), n*sizeof(int64_t), source.offset + done*sizeof(int64_t), source.name); + for (size_t i = 0; i < n; ++i) { + if (input[i] < INT32_MIN || input[i] > INT32_MAX) fail("I64 routing id out of I32 range in " + source.name); + output[i] = static_cast(input[i]); + } + fwrite_exact(out, output.data(), n*sizeof(int32_t), source.name); + done += n; + } +} + +// A dense matrix (FP8 with block scales, or BF16) quantized row by row to +// spec.type: ROCmFP4 for the projections, Q6_K for the token embedding. +void write_dense(FILE * out, const TensorSpec & spec) { + const StEntry & weight = *spec.source; + const uint32_t rows = static_cast(weight.shape[0]); + const uint32_t cols = static_cast(weight.shape[1]); + FileDescriptor wf(weight.path); + std::vector scales; + std::unique_ptr sf; + uint32_t scale_cols = 0; + if (spec.scale) { + sf = std::make_unique(spec.scale->path); + scales.resize(spec.scale->size); + pread_exact(sf->fd, scales.data(), scales.size(), spec.scale->offset, spec.scale->name); + scale_cols = static_cast(spec.scale->shape[1]); + } + const size_t in_bytes = spec.scale ? cols : cols * 2; + std::vector input(in_bytes); + std::vector values(cols); + const size_t row_bytes = ggml_row_size(spec.type, cols); + std::vector output(row_bytes); + for (uint32_t row = 0; row < rows; ++row) { + pread_exact(wf.fd, input.data(), input.size(), weight.offset + static_cast(row)*in_bytes, weight.name); + for (uint32_t col = 0; col < cols; ++col) { + float decoded; + if (spec.scale) { + decoded = fp8_e4m3fn(input[col])*fp8_e8m0(scales[(row/128)*scale_cols + col/128]); + } else { + decoded = bf16_to_float(static_cast(input[2*col] | (input[2*col + 1] << 8))); + } + if (!std::isfinite(decoded)) { + fail("non-finite weight in " + weight.name + " at row " + std::to_string(row) + " col " + std::to_string(col)); + } + values[col] = decoded; + } + if (spec.type == GGML_TYPE_Q4_0_ROCMFP4_FAST) { + rocmfp4_quantize_row_q4_0_fast_ref(values.data(), reinterpret_cast(output.data()), cols); + } else if (ggml_quantize_chunk(spec.type, values.data(), output.data(), 0, 1, cols, nullptr) != row_bytes) { + fail("quantizer wrote an unexpected row size for " + weight.name); + } + fwrite_exact(out, output.data(), output.size(), weight.name); + } +} + +const LayerCalibration & find_calibration(const std::vector & all, int layer) { + const auto it = std::find_if(all.begin(), all.end(), [layer](const LayerCalibration & c) { return c.layer == layer; }); + if (it == all.end()) fail("internal error: missing calibration for layer " + std::to_string(layer)); + return *it; +} + +void write_expert_tensor(FILE * out, const SafeTensorSet & source, + const LayerCalibration & calibration, uint32_t experts, + const ExpertRecipe & recipe, const std::optional & imatrix) { + const TensorShape expected = recipe.books == BookSource::GateUpJoint + ? calibration.gate_up_shape : calibration.down_shape; + const CodebookRegistry & registry = recipe.books == BookSource::GateUpJoint + ? calibration.gate_up : calibration.down; + const std::string target = target_expert_name(calibration.layer, recipe); + if (recipe.qtype != GGML_TYPE_Q2_1_ROCMFP2_MIX && recipe.qtype != GGML_TYPE_Q3_1_ROCMFP3_MIX) { + fail("recipe table contains unsupported qtype"); + } + const std::function(uint32_t)> encode_expert = [&](uint32_t expert) { + const TensorShape shape = validate_expert_source(source, calibration.layer, expert, recipe); + if (shape.in != expected.in || shape.out != expected.out) fail("expert shape drift in " + target); + const StEntry & w = source.at(source_expert_name(calibration.layer, expert, recipe, "weight")); + const StEntry & s = source.at(source_expert_name(calibration.layer, expert, recipe, "scale")); + OpenTensorPair input(w, s); + const auto & books = registry.experts.at(expert); + const float * importance = require_imatrix(imatrix, target, expected.in, expert, experts); + std::vector packed, scales; + std::vector values; + std::vector q2(expected.in/kBlock); + std::vector q3(expected.in/kBlock); + const size_t row_bytes = recipe.qtype == GGML_TYPE_Q2_1_ROCMFP2_MIX + ? q2.size()*sizeof(q2[0]) : q3.size()*sizeof(q3[0]); + std::vector bytes; + bytes.reserve(static_cast(shape.out) * row_bytes); + for (uint32_t row = 0; row < shape.out; ++row) { + decode_expert_row(input, row, shape.in, packed, scales, values); + const uint8_t * encoded = nullptr; + if (recipe.qtype == GGML_TYPE_Q2_1_ROCMFP2_MIX) { + if (!rocmfpx_quantize_row_fp2_mix_ref(values.data(), q2.data(), shape.in, + books.data(), importance)) { + fail("qtype-106 reference encoder rejected " + w.name); + } + encoded = reinterpret_cast(q2.data()); + } else { + if (!rocmfpx_quantize_row_fp3_mix_ref(values.data(), q3.data(), shape.in, + books.data(), importance)) { + fail("qtype-105 reference encoder rejected " + w.name); + } + encoded = reinterpret_cast(q3.data()); + } + bytes.insert(bytes.end(), encoded, encoded + row_bytes); + } + return bytes; + }; + const std::function &)> write = [&](uint32_t expert, std::vector & bytes) { + fwrite_exact(out, bytes.data(), bytes.size(), target); + std::cerr << "[encode] " << target << " expert " << (expert + 1) << "/" << experts << "\n"; + }; + for_each_expert_ordered>(experts, encode_expert, write); +} + +std::vector validate_input_layout( + const SafeTensorSet & source, uint32_t layers, uint32_t experts) { + std::vector result; + result.reserve(layers); + for (uint32_t layer = 0; layer < layers; ++layer) { + LayerCalibration current; + current.layer = static_cast(layer); + for (uint32_t expert = 0; expert < experts; ++expert) { + for (const ExpertRecipe * recipe_ptr : layer_recipes(static_cast(layer))) { + const ExpertRecipe & recipe = *recipe_ptr; + const TensorShape shape = validate_expert_source(source, layer, expert, recipe); + TensorShape & expected = recipe.books == BookSource::GateUpJoint + ? current.gate_up_shape : current.down_shape; + if (expected.in == 0) expected = shape; + if (shape.in != expected.in || shape.out != expected.out) { + fail("expert shape drift during input validation at layer " + std::to_string(layer)); + } + } + } + result.push_back(std::move(current)); + } + return result; +} + +std::vector calibrate( + const SafeTensorSet & source, uint32_t layers, uint32_t experts, + const std::optional & imatrix) { + std::vector result; + result.reserve(layers); + for (uint32_t layer = 0; layer < layers; ++layer) { + LayerCalibration current; + current.layer = static_cast(layer); + current.gate_up.levels = kGuLevels; + current.down.levels = down_recipe(static_cast(layer)).levels; + struct ExpertFit { + TensorShape gate_shape, down_shape; + std::vector gate_up, down; + std::vector repairs; + }; + const std::function fit_expert = [&](uint32_t expert) { + ExpertFit fit; + HistogramFitter gate_up_fitter; + for (const ExpertRecipe & recipe : kExpertRecipes) { + if (recipe.books != BookSource::GateUpJoint) continue; + const TensorShape shape = validate_expert_source(source, layer, expert, recipe); + if (fit.gate_shape.in == 0) fit.gate_shape = shape; + if (shape.in != fit.gate_shape.in || shape.out != fit.gate_shape.out) { + fail("qtype-106 gate/up shape mismatch at layer " + std::to_string(layer)); + } + const std::string target = target_expert_name(layer, recipe); + const float * importance = require_imatrix(imatrix, target, shape.in, expert, experts); + add_expert_to_fitter(source, layer, expert, recipe, importance, gate_up_fitter); + } + HistogramFitter down_fitter; + const ExpertRecipe & down = down_recipe(static_cast(layer)); + fit.down_shape = validate_expert_source(source, layer, expert, down); + const float * down_importance = require_imatrix( + imatrix, target_expert_name(layer, down), fit.down_shape.in, expert, experts); + add_expert_to_fitter(source, layer, expert, down, down_importance, down_fitter); + const std::string label = "layer=" + std::to_string(layer) + " expert=" + std::to_string(expert); + fit.gate_up = gate_up_fitter.fit(kGuLevels, label + " gate_up", &fit.repairs); + fit.down = down_fitter.fit(static_cast(down.levels), label + " down", &fit.repairs); + return fit; + }; + const std::function keep = [&](uint32_t expert, ExpertFit & fit) { + if (current.gate_up_shape.in == 0) current.gate_up_shape = fit.gate_shape; + if (current.down_shape.in == 0) current.down_shape = fit.down_shape; + if (current.gate_up_shape.in != fit.gate_shape.in || current.gate_up_shape.out != fit.gate_shape.out || + current.down_shape.in != fit.down_shape.in || current.down_shape.out != fit.down_shape.out) { + fail("expert shapes vary within layer " + std::to_string(layer)); + } + current.gate_up.experts.push_back(std::move(fit.gate_up)); + current.down.experts.push_back(std::move(fit.down)); + for (auto & stamp : fit.repairs) current.repairs.push_back(std::move(stamp)); + std::cerr << "[calibration] layer " << layer << " expert " << (expert + 1) + << "/" << experts << " fitted joint gate/up and down codebooks\n"; + }; + for_each_expert_ordered(experts, fit_expert, keep); + result.push_back(std::move(current)); + } + return result; +} + +void compare_raw_passthrough(int output_fd, uint64_t output_offset, const StEntry & input) { + FileDescriptor source_fd(input.path); + std::vector a(std::min(input.size, kCopyChunk)); + std::vector b(a.size()); + uint64_t done = 0; + while (done < input.size) { + const size_t n = static_cast(std::min(a.size(), input.size - done)); + pread_exact(source_fd.fd, a.data(), n, input.offset + done, input.name + " source verification"); + pread_exact(output_fd, b.data(), n, output_offset + done, input.name + " output verification"); + if (std::memcmp(a.data(), b.data(), n) != 0) fail("lossless pass-through verification failed for " + input.name); + done += n; + } +} + +std::vector read_file(const fs::path & path) { + std::ifstream in(path, std::ios::binary | std::ios::ate); + if (!in) fail("cannot open " + path.string()); + const auto end = in.tellg(); + if (end < 0) fail("cannot size " + path.string()); + std::vector out(static_cast(end)); + in.seekg(0); + if (!out.empty() && !in.read(reinterpret_cast(out.data()), out.size())) fail("cannot read " + path.string()); + return out; +} + +void verify_artifact(const fs::path & output, + const std::vector & plan, + const std::vector & expected_p4, + const std::vector & expected_gumix) { + ggml_context * meta = nullptr; + gguf_init_params params{}; + params.no_alloc = true; + params.ctx = &meta; + gguf_context * ctx = gguf_init_from_file(output.c_str(), params); + if (!ctx) fail("GGUF verification could not parse " + output.string()); + const uint64_t file_size = fs::file_size(output); + const uint64_t data_offset = gguf_get_data_offset(ctx); + const int64_t p4_key = gguf_find_key(ctx, "deepseek4.p4mix.sidecar"); + if (expected_p4.empty() ? p4_key >= 0 : + p4_key < 0 || gguf_get_kv_type(ctx, p4_key) != GGUF_TYPE_ARRAY || + gguf_get_arr_type(ctx, p4_key) != GGUF_TYPE_UINT8 || + gguf_get_arr_n(ctx, p4_key) != expected_p4.size() || + std::memcmp(gguf_get_arr_data(ctx, p4_key), expected_p4.data(), expected_p4.size()) != 0) { + fail("embedded deepseek4.p4mix.sidecar verification failed"); + } + const int64_t gu_key = gguf_find_key(ctx, "deepseek4.gumix.sidecar"); + if (gu_key < 0 || gguf_get_kv_type(ctx, gu_key) != GGUF_TYPE_ARRAY || + gguf_get_arr_type(ctx, gu_key) != GGUF_TYPE_UINT8 || + gguf_get_arr_n(ctx, gu_key) != expected_gumix.size() || + std::memcmp(gguf_get_arr_data(ctx, gu_key), expected_gumix.data(), expected_gumix.size()) != 0) { + fail("embedded deepseek4.gumix.sidecar verification failed"); + } + + FileDescriptor output_fd(output); + for (const TensorSpec & spec : plan) { + const int64_t id = gguf_find_tensor(ctx, spec.name.c_str()); + if (id < 0) fail("output is missing tensor " + spec.name); + if (gguf_get_tensor_type(ctx, id) != spec.type) fail("output qtype mismatch for " + spec.name); + const uint64_t offset = data_offset + gguf_get_tensor_offset(ctx, id); + const uint64_t size = gguf_get_tensor_size(ctx, id); + if (offset > file_size || size > file_size - offset) fail("output tensor is out of file bounds: " + spec.name); + if (spec.producer == Producer::Raw) compare_raw_passthrough(output_fd.fd, offset, *spec.source); + } + if (gguf_get_n_tensors(ctx) != static_cast(plan.size())) fail("unexpected tensor count in output"); + gguf_free(ctx); + if (meta) ggml_free(meta); + std::cerr << "[verify] parsed GGUF, checked qtypes/bounds, exact sidecars, and raw pass-through bytes\n"; +} + +void write_gguf(const Options & options, const SafeTensorSet & source, + const std::vector & calibration, + uint32_t layers, uint32_t experts, const std::optional & imatrix) { + if (!options.force && fs::exists(options.output)) fail("output exists: " + options.output.string()); + // A loose sidecar beside the file would be read instead of nothing, but + // the embedded copy is preferred; remove a stale one so there is one source. + const fs::path stale_gumix = options.output.string() + ".gumix.bin"; + if (fs::exists(stale_gumix)) fs::remove(stale_gumix); + if (!options.output.parent_path().empty()) fs::create_directories(options.output.parent_path()); + const fs::path temporary = options.output.string() + ".partial"; + if (fs::exists(temporary)) fs::remove(temporary); + + const std::vector p4 = make_p4_blob(calibration, experts); + const std::vector gumix = make_gumix_blob(calibration, experts); + const std::vector plan = make_plan(source, calibration, experts, options.experts_only); + + gguf_context * ctx = gguf_init_empty(); + if (!ctx) fail("gguf_init_empty failed"); + const std::string calibration_note = options.imatrix + ? "importance-matrix weighted: " + options.imatrix->filename().string() : std::string(); + set_model_metadata(ctx, source, layers, experts, options.absmax_only, + options.experts_only, p4, gumix, calibration_note); + std::vector repair_stamps; + for (const auto & layer : calibration) + for (const auto & stamp : layer.repairs) repair_stamps.push_back(stamp.c_str()); + gguf_set_val_str(ctx, "deepseek4.mix.codebook_repair", "bf16-epsilon-v1"); + gguf_set_arr_str(ctx, "deepseek4.mix.codebook_repairs", repair_stamps.data(), repair_stamps.size()); + std::vector> descriptors; + descriptors.reserve(plan.size()); + for (const TensorSpec & spec : plan) { + descriptors.push_back(make_tensor_descriptor(spec)); + gguf_add_tensor(ctx, descriptors.back().get()); + } + if (!gguf_write_to_file(ctx, temporary.c_str(), true)) fail("failed writing GGUF metadata"); + FILE * out = std::fopen(temporary.c_str(), "ab"); + if (!out) fail("cannot append tensor data to " + temporary.string()); + const off_t initial_position = ::ftello(out); + if (initial_position < 0) fail("cannot determine GGUF data offset"); + const uint64_t data_offset = static_cast(initial_position); + for (const TensorSpec & spec : plan) { + const int64_t id = gguf_find_tensor(ctx, spec.name.c_str()); + if (id < 0) fail("internal GGUF tensor lookup failed for " + spec.name); + const uint64_t expected = data_offset + gguf_get_tensor_offset(ctx, id); + const off_t position = ::ftello(out); + if (position < 0 || static_cast(position) > expected) { + fail("GGUF stream offset overran " + spec.name + " (position=" + + std::to_string(position) + " expected=" + std::to_string(expected) + ")"); + } + std::array zeros{}; + uint64_t gap = expected - static_cast(position); + while (gap) { + const size_t n = static_cast(std::min(gap, zeros.size())); + fwrite_exact(out, zeros.data(), n, "GGUF alignment"); + gap -= n; + } + const off_t before = ::ftello(out); + if (spec.producer == Producer::Raw) { + copy_raw(out, *spec.source); + } else if (spec.producer == Producer::Dense) { + write_dense(out, spec); + } else if (spec.producer == Producer::Int64ToInt32) { + write_int64_to_int32(out, *spec.source); + } else { + write_expert_tensor(out, source, find_calibration(calibration, spec.layer), + experts, *spec.recipe, imatrix); + } + const off_t after = ::ftello(out); + if (before < 0 || after < before || static_cast(after - before) != gguf_get_tensor_size(ctx, id)) { + fail("producer wrote wrong byte count for " + spec.name); + } + } + const bool flush_failed = std::fflush(out) != 0; + const bool sync_failed = !flush_failed && ::fsync(::fileno(out)) != 0; + const bool close_failed = std::fclose(out) != 0; + if (flush_failed || sync_failed || close_failed) { + fail("failed finalizing " + temporary.string()); + } + gguf_free(ctx); + + if (options.force && fs::exists(options.output)) fs::remove(options.output); + fs::rename(temporary, options.output); + verify_artifact(options.output, plan, p4, gumix); +} + +} // namespace + +int main(int argc, char ** argv) { + try { + static_assert(static_cast(GGML_TYPE_Q3_1_ROCMFP3_MIX) == static_cast(kQtypeP4Mix)); + static_assert(static_cast(GGML_TYPE_Q2_1_ROCMFP2_MIX) == static_cast(kQtypeGuMix)); + const Options options = parse_options(argc, argv); + g_fp2_down_layers = parse_layer_set(options.down_fp2_layers); + g_threads = options.threads > 0 ? static_cast(options.threads) + : std::max(1u, std::thread::hardware_concurrency()); + SafeTensorSet source(options.input); + const uint32_t source_layers = config_u32(source.config(), "num_hidden_layers"); + const uint32_t source_experts = config_u32(source.config(), "n_routed_experts"); + const uint32_t layers = options.layer_count < 0 ? source_layers : static_cast(options.layer_count); + const uint32_t experts = options.expert_limit < 0 ? source_experts : static_cast(options.expert_limit); + if (layers == 0 || layers > source_layers || experts == 0 || experts > source_experts) { + fail("selected layer/expert range exceeds config.json"); + } + if (!options.experts_only && (layers != source_layers || experts != source_experts)) { + fail("layer/expert limits are permitted only with --experts-only smoke artifacts"); + } + if (options.validate_input_only) { + const auto layout = validate_input_layout(source, layers, experts); + const auto plan = make_plan(source, layout, experts, options.experts_only); + std::cerr << "[validate-input] PASS: " << plan.size() + << " loader and pass-through tensors; all expert sources/dtypes/shapes validated\n"; + return 0; + } + std::optional imatrix; + if (options.imatrix) { + imatrix = load_imatrix(*options.imatrix); + } else { + std::cerr << "[calibration] WARNING: ABSMAX-ONLY LOWER QUALITY mode explicitly selected; no imatrix weighting\n"; + } + std::cerr << "[plan] CPU-only conversion layers=0.." << (layers - 1) + << " experts=0.." << (experts - 1) + << (options.experts_only ? " experts-only smoke artifact" : " complete text+vision artifact") << "\n"; + const auto calibration = calibrate(source, layers, experts, imatrix); + write_gguf(options, source, calibration, layers, experts, imatrix); + std::cerr << "[done] " << options.output << "\n"; + return 0; + } catch (const std::exception & e) { + std::cerr << "ds4_mix_converter: ERROR: " << e.what() << "\n"; + return 1; + } +} diff --git a/server/tools/export_ds4v_mmproj.py b/server/tools/export_ds4v_mmproj.py new file mode 100755 index 000000000..9921a9ff2 --- /dev/null +++ b/server/tools/export_ds4v_mmproj.py @@ -0,0 +1,508 @@ +#!/usr/bin/env python3 +"""Extract the DeepSeek-V4 vision projector from parent safetensors. + +The exporter is intentionally standard-library-only. It copies the BF16 tensor +payloads byte for byte and writes a small GGUF v3 header around them; it never +loads, converts, or quantizes tensor values. +""" + +import argparse +import json +import os +import struct +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, List, Mapping, Sequence, Tuple + + +GGUF_MAGIC = b"GGUF" +GGUF_VERSION = 3 +GGUF_ALIGNMENT = 32 +GGML_TYPE_BF16 = 30 + +GGUF_TYPE_UINT32 = 4 +GGUF_TYPE_FLOAT32 = 6 +GGUF_TYPE_STRING = 8 +GGUF_TYPE_ARRAY = 9 + +MAX_INDEX_BYTES = 64 * 1024 * 1024 +MAX_CONFIG_BYTES = 1024 * 1024 +MAX_SAFETENSORS_HEADER_BYTES = 128 * 1024 * 1024 +COPY_CHUNK_BYTES = 8 * 1024 * 1024 + + +class ExportError(RuntimeError): + """A malformed source or unsafe output prevented export.""" + + +@dataclass(frozen=True) +class SourceTensor: + name: str + path: Path + shape: Tuple[int, ...] + dtype: str + file_offset: int + nbytes: int + file_size: int + device: int + inode: int + mtime_ns: int + + +@dataclass(frozen=True) +class OutputTensor: + source: SourceTensor + data_offset: int + + +def _is_projector_tensor(name: str) -> bool: + return name.startswith("vision.") or name.startswith("aligner.") or name.startswith("image_") + + +def expected_shapes() -> Dict[str, Tuple[int, ...]]: + """Return the complete, source-name-preserving DS4V projector inventory.""" + shapes = { + "aligner.w1.bias": (4096,), + "aligner.w1.weight": (4096, 9216), + "aligner.w2.bias": (4096,), + "aligner.w2.weight": (4096, 4096), + "image_end": (4096,), + "image_newline": (4096,), + "image_pad": (4096,), + "image_start": (4096,), + "vision.norm.weight": (1024,), + "vision.patch_embed.proj.bias": (1024,), + "vision.patch_embed.proj.weight": (1024, 3 * 14 * 14), + } + for block in range(32): + prefix = "vision.blocks.%d" % block + shapes.update({ + prefix + ".attn.wo.bias": (1024,), + prefix + ".attn.wo.weight": (1024, 1024), + prefix + ".attn.wqkv.bias": (3 * 1024,), + prefix + ".attn.wqkv.weight": (3 * 1024, 1024), + prefix + ".mlp.w1.weight": (2 * 2816, 1024), + prefix + ".mlp.w2.weight": (1024, 2816), + prefix + ".norm1.weight": (1024,), + prefix + ".norm2.weight": (1024,), + }) + return shapes + + +EXPECTED_CONFIG = { + "vision_n_layers": 32, + "vision_dim": 1024, + "vision_n_heads": 16, + "vision_inter_dim": 2816, + "vision_patch_size": 14, + "vision_rope_theta": 10000.0, + "vision_downsample_ratio": 3, + "hidden_size": 4096, + "vocab_size": 129280, + "vision_max_n_token": 384, + "vision_min_pixels": 147456, + "vision_max_wh_ratio": 8, +} + + +def metadata() -> List[Tuple[str, str, object]]: + """Metadata contract consumed by the future DS4V vision loader.""" + return [ + ("general.architecture", "string", "deepseek4_vision"), + ("general.name", "string", "DeepSeek-V4 vision projector"), + ("general.type", "string", "mmproj"), + ("general.alignment", "uint32", GGUF_ALIGNMENT), + ("deepseek4.vision.schema_version", "uint32", 1), + ("deepseek4.vision.block_count", "uint32", 32), + ("deepseek4.vision.embedding_length", "uint32", 1024), + ("deepseek4.vision.attention.head_count", "uint32", 16), + ("deepseek4.vision.attention.head_dimension", "uint32", 64), + ("deepseek4.vision.attention.rope_layout", "string", "2d-half-split-height-width"), + ("deepseek4.vision.feed_forward_length", "uint32", 2816), + ("deepseek4.vision.patch_size", "uint32", 14), + ("deepseek4.vision.rope.freq_base", "float32", 10000.0), + ("deepseek4.vision.downsample_ratio", "uint32", 3), + ("deepseek4.vision.aligner_input_length", "uint32", 9216), + ("deepseek4.vision.language_embedding_length", "uint32", 4096), + ("deepseek4.vision.vocabulary_size", "uint32", 129280), + ("deepseek4.vision.attention.layer_norm_rms_epsilon", "float32", 1e-6), + ("deepseek4.vision.image.max_tokens", "uint32", 384), + ("deepseek4.vision.image.min_pixels", "uint32", 147456), + ("deepseek4.vision.image.max_aspect_ratio", "float32", 8.0), + ("deepseek4.vision.image.normalization_mean", "float32_array", (0.5, 0.5, 0.5)), + ("deepseek4.vision.image.normalization_std", "float32_array", (0.5, 0.5, 0.5)), + ("deepseek4.vision.image.patch_layout", "string", "channel-major"), + ("deepseek4.vision.image.layout", "string", "n"), + ("deepseek4.vision.image.layout_version", "uint32", 1), + ("deepseek4.vision.image.layout_recipe", "string", "row-pair-column-interleave"), + ("deepseek4.vision.image.compression_alignment", "uint32", 4), + ("deepseek4.vision.image.sentinel_types", "string", "start,pad,image,newline,end"), + ("deepseek4.vision.image.sentinel_type_count", "uint32", 5), + ("deepseek4.vision.aligner.padding", "string", "bottom-right"), + ("deepseek4.vision.aligner.patch_layout", "string", "channel-first-unfold"), + ("deepseek4.vision.aligner.activation", "string", "gelu-exact"), + ] + + +def _unique_object(pairs: Iterable[Tuple[str, object]]) -> Dict[str, object]: + result = {} + for key, value in pairs: + if key in result: + raise ExportError("duplicate JSON key: %s" % key) + result[key] = value + return result + + +def _read_json(path: Path, limit: int, description: str) -> Mapping[str, object]: + try: + size = path.stat().st_size + except OSError as exc: + raise ExportError("cannot stat %s %s: %s" % (description, path, exc)) from exc + if size > limit: + raise ExportError("%s exceeds %d-byte limit: %s" % (description, limit, path)) + try: + with path.open("r", encoding="utf-8") as handle: + value = json.load(handle, object_pairs_hook=_unique_object) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ExportError("cannot read %s %s: %s" % (description, path, exc)) from exc + if not isinstance(value, dict): + raise ExportError("%s must contain a JSON object: %s" % (description, path)) + return value + + +def _source_child(root: Path, name: str, description: str) -> Path: + if not name or name in (".", "..") or "/" in name or "\\" in name or Path(name).name != name: + raise ExportError("unsafe %s path: %r" % (description, name)) + candidate = root / name + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(root) + except (OSError, ValueError) as exc: + raise ExportError("%s escapes or is missing from source directory: %r" % (description, name)) from exc + if not resolved.is_file(): + raise ExportError("%s is not a regular file: %s" % (description, resolved)) + return resolved + + +def _validate_config(config: Mapping[str, object]) -> None: + for key, expected in EXPECTED_CONFIG.items(): + if key not in config: + raise ExportError("config.json is missing required key %s" % key) + actual = config[key] + if isinstance(expected, float): + if not isinstance(actual, (int, float)) or isinstance(actual, bool) or float(actual) != expected: + raise ExportError("config %s must be %r, got %r" % (key, expected, actual)) + elif actual != expected or isinstance(actual, bool): + raise ExportError("config %s must be %r, got %r" % (key, expected, actual)) + + +def _parse_safetensors_header(path: Path) -> Tuple[int, Mapping[str, object], os.stat_result]: + try: + stat_before = path.stat() + with path.open("rb") as handle: + raw_size = handle.read(8) + if len(raw_size) != 8: + raise ExportError("truncated safetensors size header: %s" % path) + header_size = struct.unpack(" MAX_SAFETENSORS_HEADER_BYTES: + raise ExportError("invalid safetensors JSON header size %d: %s" % (header_size, path)) + if 8 + header_size > stat_before.st_size: + raise ExportError("safetensors JSON header exceeds file bounds: %s" % path) + raw_header = handle.read(header_size) + stat_after = path.stat() + except OSError as exc: + raise ExportError("cannot read safetensors header %s: %s" % (path, exc)) from exc + if (stat_before.st_dev, stat_before.st_ino, stat_before.st_size, stat_before.st_mtime_ns) != ( + stat_after.st_dev, stat_after.st_ino, stat_after.st_size, stat_after.st_mtime_ns): + raise ExportError("safetensors file changed while reading its header: %s" % path) + try: + header = json.loads(raw_header, object_pairs_hook=_unique_object) + except (UnicodeError, json.JSONDecodeError) as exc: + raise ExportError("invalid safetensors JSON header %s: %s" % (path, exc)) from exc + if not isinstance(header, dict): + raise ExportError("safetensors header must be an object: %s" % path) + return header_size, header, stat_after + + +def _validate_tensor_entry(name: str, entry: object, data_bytes: int) -> Tuple[str, Tuple[int, ...], int, int]: + if not isinstance(entry, dict): + raise ExportError("tensor %s header entry must be an object" % name) + if set(entry) != {"dtype", "shape", "data_offsets"}: + raise ExportError("tensor %s header entry has unexpected fields" % name) + dtype = entry["dtype"] + shape = entry["shape"] + offsets = entry["data_offsets"] + if dtype != "BF16": + raise ExportError("tensor %s must remain BF16, got %r" % (name, dtype)) + if not isinstance(shape, list) or not shape or any( + not isinstance(dim, int) or isinstance(dim, bool) or dim <= 0 for dim in shape): + raise ExportError("tensor %s has an invalid shape %r" % (name, shape)) + if not isinstance(offsets, list) or len(offsets) != 2 or any( + not isinstance(offset, int) or isinstance(offset, bool) for offset in offsets): + raise ExportError("tensor %s has invalid data offsets %r" % (name, offsets)) + start, end = offsets + if start < 0 or end <= start or end > data_bytes: + raise ExportError("tensor %s data offsets %r exceed safetensors bounds" % (name, offsets)) + elements = 1 + for dim in shape: + elements *= dim + expected_bytes = elements * 2 + if end - start != expected_bytes: + raise ExportError("tensor %s has %d bytes; BF16 shape %r requires %d" % ( + name, end - start, shape, expected_bytes)) + return dtype, tuple(shape), start, end + + +def discover_tensors(source_dir: Path) -> List[SourceTensor]: + try: + root = source_dir.resolve(strict=True) + except OSError as exc: + raise ExportError("source directory does not exist: %s" % source_dir) from exc + if not root.is_dir(): + raise ExportError("source is not a directory: %s" % root) + + config_path = _source_child(root, "config.json", "config") + index_path = _source_child(root, "model.safetensors.index.json", "safetensors index") + _validate_config(_read_json(config_path, MAX_CONFIG_BYTES, "config")) + index = _read_json(index_path, MAX_INDEX_BYTES, "safetensors index") + weight_map = index.get("weight_map") + if not isinstance(weight_map, dict): + raise ExportError("safetensors index is missing object-valued weight_map") + if any(not isinstance(name, str) or not isinstance(shard, str) for name, shard in weight_map.items()): + raise ExportError("safetensors weight_map keys and values must be strings") + + expected = expected_shapes() + selected = {name: shard for name, shard in weight_map.items() if _is_projector_tensor(name)} + missing = sorted(set(expected) - set(selected)) + extra = sorted(set(selected) - set(expected)) + if missing or extra: + details = [] + if missing: + details.append("missing: " + ", ".join(missing[:8]) + (" ..." if len(missing) > 8 else "")) + if extra: + details.append("unexpected: " + ", ".join(extra[:8]) + (" ..." if len(extra) > 8 else "")) + raise ExportError("incomplete projector inventory (%s)" % "; ".join(details)) + + shard_paths = {} + for shard_name in sorted(set(selected.values())): + shard_paths[shard_name] = _source_child(root, shard_name, "safetensors shard") + + headers = {} + for shard_name, path in shard_paths.items(): + header_size, header, file_stat = _parse_safetensors_header(path) + headers[shard_name] = (header_size, header, file_stat) + indexed_here = {name for name, mapped_shard in selected.items() if mapped_shard == shard_name} + present_here = {name for name in header if name != "__metadata__" and _is_projector_tensor(name)} + if present_here != indexed_here: + missing_here = sorted(indexed_here - present_here) + unindexed_here = sorted(present_here - indexed_here) + raise ExportError("projector/index disagreement in %s (missing=%r, unindexed=%r)" % ( + shard_name, missing_here[:8], unindexed_here[:8])) + + tensors = [] + ranges_by_shard = {} + for name in sorted(expected): + shard_name = selected[name] + path = shard_paths[shard_name] + header_size, header, file_stat = headers[shard_name] + if name not in header: + raise ExportError("index tensor %s is absent from %s" % (name, shard_name)) + data_bytes = file_stat.st_size - 8 - header_size + dtype, shape, start, end = _validate_tensor_entry(name, header[name], data_bytes) + if shape != expected[name]: + raise ExportError("tensor %s shape must be %r, got %r" % (name, expected[name], shape)) + ranges_by_shard.setdefault(shard_name, []).append((start, end, name)) + tensors.append(SourceTensor( + name=name, + path=path, + shape=shape, + dtype=dtype, + file_offset=8 + header_size + start, + nbytes=end - start, + file_size=file_stat.st_size, + device=file_stat.st_dev, + inode=file_stat.st_ino, + mtime_ns=file_stat.st_mtime_ns, + )) + + for shard_name, ranges in ranges_by_shard.items(): + previous_end = -1 + previous_name = "" + for start, end, name in sorted(ranges): + if start < previous_end: + raise ExportError("overlapping projector tensors in %s: %s and %s" % ( + shard_name, previous_name, name)) + previous_end = end + previous_name = name + return tensors + + +def _pack_string(value: str) -> bytes: + raw = value.encode("utf-8") + return struct.pack(" bytes: + result = bytearray(_pack_string(key)) + if kind == "uint32": + result += struct.pack(" int: + return (value + alignment - 1) // alignment * alignment + + +def _build_header(tensors: Sequence[SourceTensor]) -> Tuple[bytes, List[OutputTensor]]: + metadata_entries = metadata() + output_tensors = [] + offset = 0 + tensor_info = bytearray() + for tensor in tensors: + output_tensors.append(OutputTensor(tensor, offset)) + tensor_info += _pack_string(tensor.name) + tensor_info += struct.pack(" None: + current = os.fstat(handle.fileno()) + actual = (current.st_dev, current.st_ino, current.st_size, current.st_mtime_ns) + expected = (tensor.device, tensor.inode, tensor.file_size, tensor.mtime_ns) + if actual != expected: + raise ExportError("source shard changed after validation: %s" % tensor.path) + + +def _copy_tensor(output, tensor: SourceTensor) -> None: + try: + with tensor.path.open("rb") as source: + _check_source_identity(source, tensor) + source.seek(tensor.file_offset) + remaining = tensor.nbytes + while remaining: + chunk = source.read(min(remaining, COPY_CHUNK_BYTES)) + if not chunk: + raise ExportError("source tensor became truncated: %s" % tensor.name) + output.write(chunk) + remaining -= len(chunk) + _check_source_identity(source, tensor) + except OSError as exc: + raise ExportError("cannot copy tensor %s from %s: %s" % (tensor.name, tensor.path, exc)) from exc + padding = _align(tensor.nbytes) - tensor.nbytes + if padding: + output.write(bytes(padding)) + + +def _validate_output_path(output_path: Path) -> Path: + if output_path.name in ("", ".", ".."): + raise ExportError("output must name a GGUF file") + try: + parent = output_path.parent.resolve(strict=True) + except OSError as exc: + raise ExportError("output directory does not exist: %s" % output_path.parent) from exc + if not parent.is_dir(): + raise ExportError("output parent is not a directory: %s" % parent) + output = parent / output_path.name + if os.path.lexists(str(output)): + raise ExportError("refusing to overwrite existing output: %s" % output) + return output + + +def export_mmproj(source_dir: Path, output_path: Path) -> Tuple[int, int]: + tensors = discover_tensors(source_dir) + header, output_tensors = _build_header(tensors) + output = _validate_output_path(output_path) + + temp_fd = -1 + temp_name = "" + linked = False + try: + temp_fd, temp_name = tempfile.mkstemp( + prefix=".%s." % output.name, + suffix=".tmp", + dir=str(output.parent), + ) + with os.fdopen(temp_fd, "wb") as handle: + temp_fd = -1 + handle.write(header) + for output_tensor in output_tensors: + _copy_tensor(handle, output_tensor.source) + handle.flush() + os.fsync(handle.fileno()) + + # Hard-linking is an atomic no-overwrite publish on the same filesystem. + # Readers can observe only the complete, fsynced file. + os.link(temp_name, output) + linked = True + os.unlink(temp_name) + temp_name = "" + directory_fd = os.open(str(output.parent), os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + except FileExistsError as exc: + raise ExportError("refusing to overwrite existing output: %s" % output) from exc + except OSError as exc: + if linked: + raise ExportError("output was published but final directory sync failed: %s" % exc) from exc + raise ExportError("cannot write output %s: %s" % (output, exc)) from exc + finally: + if temp_fd >= 0: + os.close(temp_fd) + if temp_name: + try: + os.unlink(temp_name) + except FileNotFoundError: + pass + + return len(tensors), sum(tensor.nbytes for tensor in tensors) + + +def _parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Losslessly extract the DeepSeek-V4 vision projector into standalone GGUF v3.") + parser.add_argument("source_dir", type=Path, + help="parent model directory containing config.json, index, and shards") + parser.add_argument("output", type=Path, help="new output GGUF path; existing paths are refused") + return parser.parse_args(argv) + + +def main(argv: Sequence[str] = ()) -> int: + args = _parse_args(argv or sys.argv[1:]) + try: + tensor_count, payload_bytes = export_mmproj(args.source_dir, args.output) + except ExportError as exc: + print("error: %s" % exc, file=sys.stderr) + return 1 + print("exported %d BF16 tensors (%d payload bytes) to %s" % ( + tensor_count, payload_bytes, args.output)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())