diff --git a/daslib/fio.das b/daslib/fio.das index 65d1a06be1..2b3e032cac 100644 --- a/daslib/fio.das +++ b/daslib/fio.das @@ -692,8 +692,7 @@ def rmdir_rec_result(path : string) : fs_result_bool { def run_and_capture(args : array; var output : string&; timeout_sec : float = 0.0) : int { //! Run an external command and capture its stdout+stderr (merged into one pipe by the underlying ``popen_argv``). Returns the process exit code; - //! -1 means the spawn itself failed. No shell is involved, but on Windows the program path (``args[0]``) must use backslashes — - //! CreateProcess does not resolve ``bin/daslang``-style forward-slash relative paths (probe-verified); ``replace(exe, "/", "\\")`` first. + //! -1 means the spawn itself failed. No shell is involved, and a forward-slash ``args[0]`` spawns on every platform (``popen_argv`` hands Windows the backslash spelling). var captured : string let exit_code = unsafe(popen_argv(args, timeout_sec, $(f) { if (f != null) { diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index 2bbde3e57c..17a57f4708 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -55,8 +55,11 @@ re-transcoding `$LCPP/src/unicode-data.cpp`). MoE region split. - `ARCHITECTURE_MEDIA.md` - sec.2.13-2.16: the padded tower GEMM widths, the family GPU hooks, the tower weight lane, and the plain-Model ASR decoders. -- `ARCHITECTURE_MEASUREMENT.md` - sec.2.5, 2.10, 2.20: the benchmark rig, the tune gate, and the - sanctioned instrumentation rails. +- `ARCHITECTURE_MEASUREMENT.md` - sec.2.5, 2.10, 2.20, 2.21, 2.26-2.27: the benchmark rig, the tune + gate, the sanctioned instrumentation rails, kernel-race fidelity, the gemv's own tune seat, and + the CPU kernel bench's fixture conditions. +- `ARCHITECTURE_CPU_KERNELS.md` - sec.2.22-2.24: the sub-block-packed k3/k6 planes, the grid + formats' panel and row-group decodes, and the VBMI symbol lattice. ## 3. Inherited invariants diff --git a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md new file mode 100644 index 0000000000..9a32bf3b44 --- /dev/null +++ b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md @@ -0,0 +1,45 @@ +# dasLLAMA architecture - the CPU kernel planes and decodes + +Companion of `ARCHITECTURE.md` (contract: `../../ARCHITECTURE_COMMON.md`). Section 2 here continues +the mechanism numbering; each section is cited by the code that embodies it. + +## 2. Mechanisms + +### 2.22 The k3 and k6 planes are packed per sub-block {#kq-subblock-planes} + +A k6 grp plane's qh columns `2blk` and `2blk + 1` carry one sub-block's four `j` sites, each as +a 2-bit field at bit `2j`; the disk byte `h*32 + half*16 + j*4 + t` feeds sub-blocks `4h..` at bit +`2b`. k3 packs the same way - qs columns `2blk + half` carry the sub-block's four `j` sites at `2j`, +and the hmask column `blk` carries its eight sites at bit `s` (lo at `j`, hi at `4 + j`). One +sub-block's decode then costs two loads for k6 and three for k3, and nothing loaded lives past it; +the row-interleaved disk order cost one load per `j`. The layout is CPU-flavor: the `.dlim` a box +bakes is for the hardware that runs it, so a CPU plane owes nothing to the GPU tiers' shapes. +`IMAGE_VERSION` 28 is this layout. + +### 2.23 A grid format's CPU gemv decodes as a panel or as row groups {#grid-decode-forms} + +Five formats (iq3s, iq3xxs, iq2s, iq2xs, iq2xxs) have two gemv decode forms. The PANEL form gathers +a superblock into an alloca panel first and reads packed positions through one dword load per +4-byte column, the four positions of a column sharing the load. The ROW-GROUP form composes a +weight-width vector straight from the grid words - width/64 rows x 8 weights, one u64 grid entry +per iq2 row - and reverts to byte loads, because the column dword read only pays inside the panel. +The sdot lattice always takes row groups; on x86 the panel's latency chain does not scale with the +core, so `x86-vnni512` takes row groups for iq2xxs and `x86-amx` for iq2xxs and iq3xxs, everything +else the panel. A VBMI seat (sec.2.24) takes row groups unconditionally. + +### 2.24 The VBMI symbol lattice {#vbmi-lattice} + +On a zmm VBMI target a grid block decodes as row groups through a symbol lattice. Every grid byte +comes from a tiny alphabet (three symbols for the iq2 family, eight for iq3), so the grid is baked +as two compact code planes - entry `e`'s low and high half, four 2-bit symbols each for iq2, two +3-bit for iq3 - plus the alphabet as a per-lane `vpshufb` table and `ksigns_iq2xs` whole (128 bytes: +exactly the two registers one `VPERMI2B` indexes). Per format the block's index bytes gather into one +64-lane vector per column (lane `r*4 + position`) and look up in the code planes - `VPERMI2B` per 128 +entries, index bit 7 blends the pairs, the 9th and 10th index bits arrive as lane masks (iq3s and iq2s +from the row's qh byte, iq2xs from bit 0 of its u16 word's high byte). The row's sign bytes land in +the same lane layout: the plane's own column for iq3s and iq2s, one ksigns `VPERMI2B` over the 7-bit +codes for the rest. Per row group and weight octet a constant two-source shuffle places each row's +code bytes in its qword, `VPMULTISHIFTQB` spreads the symbols into bytes, one `vpshufb` maps them to +magnitudes, and the signs ride the activation copy as a mask `(x ^ m) - m`. The lattice row shares +its tile body and planes with the 512/mr16 row, so only the gemv differs - what the gemv's own seat +(`ARCHITECTURE_MEASUREMENT.md` sec.2.26) races. diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index f46cc5233c..f2465a8b33 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -160,3 +160,32 @@ no model, no tuner, every arm gated against a CPU plane-dequant oracle before it chain every dispatch through ONE shared output buffer on purpose - the serialized regime is the instrument's probe shape, imitating the reference tool it is compared against - and its numbers reach the engine only through a human porting decision, never a minted crown. + +### 2.26 The gemv takes its own tune seat {#gemv-seat} + +A kq family's manifest entry is its tile-best row, and the gemv gets a SECOND entry when a different +row serves the streamed decode better. Only same-mr rows can differ, because the layout companion +pins the plane's interleave; of those the two best by tile time race, the winner takes the gemv only +by the margin over the tile winner's own gemv, and the incumbent keeps a tie. Every family's perm grid +therefore carries a 256-wide `mr = 16` alternate beside its 512-wide tile crown. The seat is decided +at the engine's decode shape - a DRAM-bound plane streamed by every lane through the engine's own +splitter - because the engine's row length moves the answer (k3 on Granite Rapids: the 256 seat wins +at n=2048 and loses at 14336 - `benchmarks/matmul/kq_kernel_bench.das`, tune mode, seats pinned, d=32768). The seat fixture is a 512-row build at the ffn width tiled 320 times, +past the largest L3 a socket lends a slice of, and the seat takes the MEDIAN of seven rounds: a round +that finds the plane in L3 must not crown it. In normal mode `llvm_tune` stamps a companion from its +own manifest entry when one exists and is a perm this box can run, else from the tile's. + +### 2.27 The CPU kernel bench's fixture conditions {#cpu-kernel-bench-fixture} + +`benchmarks/matmul/kq_kernel_bench.das` times raw kernels on synthetic planes, and three fixture +properties decide whether its numbers mean anything. Every plane of one format lives in ONE arena at +fixed offsets, staggered so no two starts share their low 12 address bits: the heap places separate +arrays at run-dependent relative addresses, and planes that alias in the L1/L2 set logic make a run's +time depend on where the heap put them. Scale planes are filled with a byte that is a normal number in +every scale form, never random bytes, because denormal math runs orders of magnitude slower. Each row +is warmed before it is timed - three unmeasured rounds solo, six dispatches per row on the team arm - +because a core ramps over several rounds and one warm call is not enough. The q8 row exists in two +flavors: f32 group scales (the engine's own quantization) and `q8s16` over binary16 scales - the +wscale_f16 rail a GGUF q8_0 tensor runs, and the like-for-like row against the reference's q8_0. +Provenance for every figure in this section: `benchmarks/matmul/kq_kernel_bench.das` under +`DAS_TUNE_MODE=tune`, one thread, its default `--fmt` / `-n` / `-d` shape. diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 5e2703b677..bff6a3b94f 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -260,6 +260,19 @@ loses cross-module inlining); the first run after a cache write pays one cold co deser re-key); QUIRK 21 still applies to emitter edits. Numbers, caveats and the invalidation ledger: `plans/jit_compile_time.md`. +**Kernel-loop invocation (adopted 2026-09-01):** a kernel spelling is raced without a model in +`benchmarks/matmul/kq_kernel_bench.das` - `DAS_TUNE_MODE=tune bin/Release/daslang.exe -jit +modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das -- --fmt --perm ` times every +row of the tile's and gemv's `_variants()` registries at one thread on synthetic planes (seconds +per try), and the reference row at the same shape is the reference exe's `test-backend-ops perf +-o MUL_MAT -p "type_a=,type_b=f32,m=4096,n=1,"` under `GGML_BENCH_THREADS=1`; +`harness/kernel_ladder.sh` runs both sides for every format and prints the box's ratio table. The +app run comes only after a spelling wins there. Procedure, fact base and work queue: +`plans/kernel_parity_pass.md`. +2026-09-01: the pass closed the M1 CPU at kernel level - all 32 ladder rows at or above the +reference (the grid decodes 1.16-1.51x, from 0.51-0.93x) - so the per-format M1 CPU tg tails stamped +below (0.51x-0.73x) predate the ARM row-group decode; re-stamp the vehicles before quoting them. + A real file whose every tensor type is now loadable (the header census script in the session scratchpad, or `harness/gguf_dump.das`), through `examples/dasLLAMA/run.das` against `simple_ids.exe` from the llama.cpp reference build for the same prompt; then `test_model_image` @@ -554,6 +567,9 @@ only the step-3 0.0267 top-2 flip vs llama.cpp. M1 benches: CPU das 897.4/56.3 v is CLOSED - and with it THE FORMAT LADDER: four-tier table zen2 2.86x/0.70x, vk 0.78x/0.70x, M1 CPU 6.41x/0.57x, Metal 0.93x/0.78x. +2026-09-01 (kernel parity pass): zen2 16t (`benchmarks/lcpp_bench.das --for-debug-purposes`, debug-jit) vs the clean-cpu `llama-bench` on the IQ2_XXS-local 1B: pp512 506.9 vs +181.2 (2.80x), tg128 88.6 vs 86.2 (1.03x, was 0.70x) - the sign column + u64 pair gemv. + ### IQ2_XS Phase A (CPU, 2026-08-31) - the ksigns u64 tier Shape: 256-superblock grid format - each of the 32 u16 qs words carries a 9-bit index into @@ -621,6 +637,9 @@ tiers. M1 16GB benches: CPU das 746.6/51.6 vs llama.cpp 144.8/101.1 (5.16x/0.51x (0.93x/0.86x - the iq2s pp class). The format is CLOSED on all four tiers; four-tier table: zen2 2.78x/0.70x, vk 0.77x/0.54x, M1 CPU 5.16x/0.51x, Metal 0.93x/0.86x. +2026-09-01 (kernel parity pass): zen2 16t (`benchmarks/lcpp_bench.das --for-debug-purposes`, debug-jit) vs the clean-cpu `llama-bench` on the IQ2_XS-local 1B: pp512 480.7 vs +174.6 (2.75x), tg128 93.8 vs 85.9 (1.09x, was 0.70x) - sign column + u64 pair + column dword read. + ### IQ2_S Phase A (CPU, 2026-08-31) - the u64-grid tier Shape: 256-superblock grid format, the first with a u64 grid - a 10-bit index (qs byte | @@ -699,6 +718,7 @@ the mradermacher i1 vehicle - IQ2_S attn x32 + IQ3_XXS/IQ3_S/Q4_K/Q5_K): | tier | pp512 das / llama.cpp | tg128 das / llama.cpp | |---|---|---| | zen2 CPU | 501.5 / 138.5 (3.62x) | 55.8 / 73.5 (0.76x) | +| zen2 CPU, 2026-09-01 kernel parity pass | 494.4 / 138.1 (3.58x) | 71.9 / 74.5 (0.97x, +-4.2) - sign column + u64 pair | | 5060 Ti Vulkan | 12099.3 / 17377.5 (0.70x - the tier class) | 292.4 / 362.5 (0.81x) | | M1 CPU | 883.3 / 413.4 (2.14x) | 53.7 / 73.9 (0.73x) | | M1 Metal | 3170.5 / 3427.2 (0.93x) | 205.8 / 220.7 (0.93x) | @@ -915,6 +935,7 @@ the local --tensor-type requant, iq3_xxs on attn_k/q + all ffn): | tier | pp512 das / llama.cpp | tg128 das / llama.cpp | |---|---|---| | zen2 CPU | 507.2 / 136.0 (3.73x) | 56.7 / 72.6 (0.78x) | +| zen2 CPU, 2026-09-01 kernel parity pass | 501.0 / 135.8 (3.69x) | 74.9 / 74.3 (1.01x) - sign column + column dword read | | 5060 Ti Vulkan | 12225.7 / 17807.7 (0.69x - the tier class) | 372.1 / 389.9 (0.95x) | | M1 CPU | 906.0 / 410.5 (2.21x) | 53.5 / 74.0 (0.72x) | | M1 Metal | 3224.0 / 3429.9 (0.94x) | 213.5 / 227.3 (0.94x) | @@ -1017,6 +1038,7 @@ embedding head is Q6_K - three formats share every decode step): | tier | pp512 das / llama.cpp | tg128 das / llama.cpp | |---|---|---| | zen2 CPU | 516.9 / 104.9 (4.93x) | 52.4 / 57.0 (0.92x) | +| zen2 CPU, 2026-09-01 kernel parity pass | 526.1 / 104.7 (5.03x) | 69.9 / 56.5 (1.24x) - the vector sign column gemv | | 5060 Ti Vulkan | 12539.6 / 17865 (0.70x) | 288.1 / 324.2 (0.89x) | | M1 CPU | 886.2 / 433.6 (2.04x) | 57.4 / 66.6 (0.86x) | | M1 Metal | 3237.6 / 3344.3 (0.97x) | 199.4 / 209.0 (0.95x) | diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md new file mode 100644 index 0000000000..7bf9b76178 --- /dev/null +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -0,0 +1,244 @@ +# How to get a sidecar: a fresh box to a committed CPU-class profile + +You have ssh into a machine nobody has tuned on. This is the walk from there to a checked-in +`performance/defaults/.tune-defaults.json` that every box of that CPU class adopts at start, +with the kernel ladder proving the kernels along the way. Every command below was run as written on +the boxes named in the last section; a step that differs per box says so. + +What a sidecar and a profile are: `skills/tune.md` (the `[tune]` framework, the per-app +`.tune.json`), `../dasLLVM/ARCHITECTURE.md` sec.3 (the knob inventory), `performance/REVIEW.md` +(what a committed profile must satisfy). Short form: the tuner races every kernel family's +`[tune_perm]` seats on the box and writes the winners into the app's sidecar; `export_tune_profile.das` +strips the box-specific rows and saves the kernel winners as the class profile; a box whose +`tune_cpu_class()` resolves to that class adopts the profile and races nothing unless its ISA unlocks +a seat the profile never saw. + +## 1. The box + +CPU classes and the AWS instance that carries each (`us-west-2`; the CLI on the zen2 box is configured): + +| class (`tune_cpu_class()`) | what gates it | instance | +|---|---|---| +| `x86-avx2` | avx2, no VNNI | any zen2/zen3 box (the dev zen2) | +| `x86-vnni512` | avx512vnni + avx512bw | `c7a.4xlarge` (EPYC zen4) | +| `x86-amx` | amx-int8 + amx-tile (+ avx512vnni) | `c7i.4xlarge` (Sapphire Rapids), `c8i` (Granite Rapids) | +| `arm-neon` / `arm-i8mm` | dotprod / i8mm | M1 / M2+, `c8g.2xlarge` (Graviton4) | + +Launch (Ubuntu 24.04, 16 vCPU, 60 GB gp3; the key pair `dasbox` and the group `dasbench-ssh` exist in +the account - the group admits the dev box's IP on port 22, `aws ec2 describe-security-groups +--group-ids sg-0dcc81f768b888767` shows the rule to update when that IP moves): + +``` +aws ec2 run-instances --image-id ami-0ba3fae0cca9442ee --instance-type c7a.4xlarge --key-name dasbox \ + --security-group-ids sg-0dcc81f768b888767 \ + --block-device-mappings '[{"DeviceName":"/dev/sda1","Ebs":{"VolumeSize":60,"VolumeType":"gp3"}}]' \ + --user-data file://aws_bootstrap.sh --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=dasbench-zen4}]' +aws ec2 describe-instances --instance-ids --query 'Reservations[0].Instances[0].[State.Name,PublicIpAddress]' --output text +ssh -i ~/.ssh/dasbox_ed25519 ubuntu@ +``` + +The AMI id is the current Canonical Ubuntu 24.04 amd64 image (`aws ec2 describe-images --owners +099720109477 --filters "Name=name,Values=ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"`, +newest by `CreationDate`). On-demand `c7a.4xlarge` is about $0.82 an hour; terminate when done (last section). + +## 2. Dependencies and the build + +The user-data script (`aws_bootstrap.sh`, reproduced in the last section) does 2-4 unattended; the +same lines by hand: + +``` +sudo apt-get update -y +sudo apt-get install -y --no-install-recommends build-essential clang cmake ninja-build git python3 pkg-config \ + libssl-dev curl ca-certificates libatomic-ops-dev libglu1-mesa-dev freeglut3-dev mesa-common-dev \ + libglfw3-dev libfreetype6-dev libudev-dev libopenal-dev libvorbis-dev libflac-dev libx11-dev \ + libxrandr-dev libxcursor-dev libxinerama-dev libxi-dev +git clone --depth 1 --recursive -b https://github.com/GaijinEntertainment/daScript.git daScript +cd daScript +CC=clang CXX=clang++ cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DDAS_GLFW_DISABLED=ON \ + -DDAS_HV_DISABLED=OFF -DDAS_SQLITE_DISABLED=OFF -DDAS_LLVM_DISABLED=OFF +cmake --build build --target daslang -j 16 +ls -la bin/daslang lib/LLVM.dll +``` + +`-DDAS_LLVM_DISABLED=OFF` is the JIT (default off). The dasLLVM module is das-only; the configure step +downloads the prebuilt `lib/LLVM.dll` for the platform (that name on every OS) - if `ls` does not show it, +nothing below runs. Only `daslang` needs building: dasLLAMA is das, its native needs are builtins. + +### A macOS box you do not own (the M4 Pro walk) + +Homebrew there may belong to another user (`/opt/homebrew` not writable) - do not chown it. CMake and +Ninja come as user-local binaries instead: the Kitware `cmake--macos-universal.tar.gz` and the +ninja-build `ninja-mac.zip` unpacked under `~/tools`, prepended to `PATH`. Apple's `/usr/bin/bison` is +2.3 and cannot read `ds2_parser.ypp`, so configure with `-DDAS_FLEX_BISON_DISABLED=ON` (the committed +generated parser is current); no OpenSSL means `-DDAS_HV_DISABLED=ON`. The reference builds with +`-DGGML_METAL=OFF -DGGML_BLAS=OFF` (CPU only). On a P+E chip give the ladder the performance cores +(`TEAM=10` on the M4 Pro), not `nproc`. + +## 3. The reference exe + +The kernel ladder's other side. Pin the same tip the dev box's tables use (`6c84c7d5d` for the +2026-09 tables); `test-backend-ops` needs the thread define (a three-line edit of `N_THREADS` in +`tests/test-backend-ops.cpp` - `harness/backend_ops_shapes.patch` is the older, wider form and no +longer applies at this tip): + +``` +git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp && git checkout 6c84c7d5d +python3 - <<'EOP' +import io +p="tests/test-backend-ops.cpp"; s=io.open(p,encoding="utf-8").read() +old="# define N_THREADS std::thread::hardware_concurrency()" +assert s.count(old)==1 +s=s.replace(old,'# define N_THREADS (getenv("GGML_BENCH_THREADS") && atoi(getenv("GGML_BENCH_THREADS")) > 0 ? (size_t) atoi(getenv("GGML_BENCH_THREADS")) : std::thread::hardware_concurrency())') +io.open(p,"w",encoding="utf-8").write(s) +EOP +cmake -B build-clean-cpu -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON -DLLAMA_CURL=OFF -DLLAMA_BUILD_TESTS=ON +cmake --build build-clean-cpu --target test-backend-ops llama-bench -j 16 +``` + +`build-clean-cpu` is the flavor the tables name: no GPU backend, no BLAS - a GPU build's +`llama-bench -ngl 0` is not CPU-only. + +The perf list has no decode row above 4096 x 14336, and on a box with a large L3 that op stays cached +between the repeated runs (the 3990X reads it at 295 GB/s with 32 threads, three times its DRAM) - a +many-lane ratio taken there flatters the reference. For the DRAM-bound row (`BIG=1` in the ladder, +d=32768) add, inside the same `for (int bs ...)` loop in `make_test_cases_perf`, after the 4096 line: +`if (bs == 1) test_cases.emplace_back(new test_mul_mat(type_a, type_b, 32768, bs, 14336, {1, 1}, {1, 1}));` + +Confirm the define is in the binary the ladder gets: `grep -c GGML_BENCH_THREADS +build-clean-cpu/bin/test-backend-ops` prints 1. A box with several llama.cpp builds is where this +bites - a binary that prints 0 runs every core and the ratio column lies; `kernel_ladder.sh` refuses it. + +## 4. Correctness before any number + +``` +cd ~/daScript +DAS_TUNE_MODE=test DAS_JOBQUE_THREADS=8 bin/daslang -jit modules/dasLLAMA/harness/gen_tune_probe.das +``` + +Every `[tune_perm]` seat of every family is stamped and checked bit-exact against its reference body; +the box's ISA is what makes seats that never ran before run here (the 512-bit VNNI seats on zen4, the +AMX leg on Intel). `ok` on every line, or the box has found a defect and the walk stops until it is fixed. + +## 5. The kernel ladder + +``` +LCPP_TBO=~/llama.cpp/build-clean-cpu/bin/test-backend-ops NTOK=512 ROUNDS=5 \ + bash modules/dasLLAMA/harness/kernel_ladder.sh all > ladder.tsv 2> ladder.err +``` + +Then the engine's shape - every lane, a weight above the box's L3 (needs the m=32768 perf row, section 3): + +``` +LCPP_TBO=~/llama.cpp/build-clean-cpu/bin/test-backend-ops NTOK=0 ROUNDS=5 TEAM=16 BIG=1 bash modules/dasLLAMA/harness/kernel_ladder.sh all > ladder_big16.tsv 2> ladder_big16.err +``` + +One thread, no model: every format's stamped kernel against the reference exe's `test-backend-ops perf` +at the same shape, decode and prefill rows, ratio = reference / ours. This is the box's table; it goes +into `plans/kernel_parity_pass.md`'s fact base (and the records store once the kernel board exists). + +## 6. The mint + +The vehicle is a public 1B GGUF pulled straight to the box - nothing copies from the dev box: + +``` +mkdir -p ~/models && cd ~/models +curl -L -o Llama-3.2-1B-Instruct-Q4_K_M.gguf https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf +cd ~/daScript +DAS_JOBQUE_THREADS=16 bin/daslang -jit modules/dasLLAMA/benchmarks/lcpp_bench.das -- -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --tune > ~/mint.log 2>&1 +``` + +808 MB, about a minute to fetch. `--tune` is the full race: every generator family +(`@tune begin name=q8_tile_gen ... @tune end ... winner= verdict=beats`), then the +`[tuned]` loop-hint kernels (`axpy`, `dot`, `rope_*`, `quantize_*`...), then +`confirm_e2e_prefill`; on the c7a.4xlarge the whole walk took 5-6 minutes. It writes +`modules/dasLLAMA/benchmarks/lcpp_bench.tune.json` beside the app and then re-launches the app to +apply it. The bench rows the re-launch would print are refused without `--for-debug-purposes` (a `-jit` +script run is not record-grade) - that refusal is expected here; the mint is done. + +Read the sidecar's provenance before anything else: + +``` +python3 -c 'import json;d=json.load(open("modules/dasLLAMA/benchmarks/lcpp_bench.tune.json"));p=d["provenance"];print(p["noise"],p["validation"],p["features"],len(d["kernels"]))' +``` + +`ok ok 49` is the pass (a few more when the harness gave a kq gemv its own seat - those entries are +named `q8_gemv_gen` and ride into the profile like any other): `noise` is the tuner's own drift verdict (a busy or thermally +unstable box says otherwise - re-mint, never edit), `validation` is every winner checked against its +fallback, `features` is the box's fingerprint of `TUNE_KNOWN_FEATURES` (zen4: +`avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni`), 49 the kernel count the scope demands. + +The export: + +``` +bin/daslang -jit modules/dasLLAMA/harness/export_tune_profile.das -- --sidecar modules/dasLLAMA/benchmarks/lcpp_bench.tune.json +git status --short modules/dasLLAMA/performance/defaults/ +``` + +It writes `performance/defaults/.tune-defaults.json` for this box's `tune_cpu_class()` +(`--class ` overrides, `--out ` relocates) with the box, engine sha and timing rows +stripped - a profile is kernel winners plus the provenance the adopt path checks. + +Compare before committing. A class that already ships a profile will differ in a few winners every +time - two mints of one class on two boxes disagree on the tie-class seats (the zen4 re-mint of +2026-09-01 flipped 4 of 49: `axpy` vec8_u2 -> vec8, `axpy_f16` -> plain, `rope_scaled_neox_tab` -> +plain, `q51q8_tile_gen` 512 -> 256-bit; the format tiles agreed). Commit a re-mint only when a seat +that carries a kernel family changed or new families exist; a tie flip is not a reason. + +``` +python3 - <<'EOP' +import json +n=json.load(open("modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json")) +import subprocess; o=json.loads(subprocess.check_output(["git","show","HEAD:modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json"])) +for k in sorted(set(n["kernels"])|set(o["kernels"])): + if n["kernels"].get(k)!=o["kernels"].get(k): print(k, o["kernels"].get(k), "->", n["kernels"].get(k)) +EOP +``` + +A class with no shipped profile yet (Intel's `x86-amx`, born 2026-09-01) needs the class first, on the +branch the box clones: `tune_cpu_class()` and the ladder in `tune_class_chain()` +(`modules/dasLLVM/daslib/llvm_tune.das`), the new class above the one it supersedes, and every feature +a `requires=` names in `TUNE_KNOWN_FEATURES` there (`amx-int8`, `amx-tile` were already listed). Until +its profile ships, a box of the new class adopts the class below it (the chain) and races only the +seats that class could not answer. Then the same mint and export produce `.tune-defaults.json`, +and that one IS committed - the Intel one differed from `x86-vnni512` in 15 of 49 winners, all +`[tuned]` loop-hint kernels preferring `vec16`, while every generator tile kept the 512-bit VNNI seat +(the AMX tiles raced and lost the q8q8 family). + +Three things the boxes taught that the walk now carries: + +- **Alignment.** Intel splits a 64-byte vector load that crosses a cache line and pays for it; AMD + barely does. The engine's image planes are page-aligned, but a bench that hands kernels 16-byte-aligned + arrays reads 2x slow on Intel and true on zen4 - `kq_kernel_bench` aligns its planes to 64 bytes. +- **Denormals.** Random bytes in a scale plane are denormals or infinities often enough to bend a + table on Intel (~100 cycles a denormal op) and not on AMD: the q8/mx4 tile rows read 6x slow until the + bench filled scale planes with a byte that is a normal number in every scale form. +- **AMX permission.** A tile instruction before the per-process `arch_prctl` grant is SIGILL. The + family's witness performs the grant; in tune mode call the witness *variants* (the AMX row does it), + as `gen_tune_probe` does - the unstamped base does not. + +## 7. What the commit must satisfy + +`modules/dasLLAMA/performance/REVIEW.md` and its `REVIEW.das` gate: a `defaults/` profile carries no +`engine_sha`, its `provenance.dasllama_version` equals `DASLLAMA_RELEASE`, its `noise` reads `ok`, +and it was minted on a quiet, session-free box - never hand-edited. Run the gate before pushing: +`bin/daslang utils/internal/preflight/main.das -- --only review-md`. + +## 8. Home, and the bill + +``` +scp -i ~/.ssh/dasbox_ed25519 ubuntu@:daScript/modules/dasLLAMA/performance/defaults/.tune-defaults.json . +scp -i ~/.ssh/dasbox_ed25519 ubuntu@:ladder.tsv ladder_.tsv +aws ec2 terminate-instances --instance-ids +``` + +## Boxes this walk ran on + +- 2026-09-01 `c7a.4xlarge` (EPYC 9R14 zen4, class `x86-vnni512`), `i-043725feb25086523`: sections 1-6 + as written; the TEST gate 65/65 ok; the re-mint agreed with the shipped profile on every format tile + and flipped four tie seats, so nothing was committed from it. Model-level sanity on the vehicle + (`lcpp_bench -m -p 512 -n 128 -r 3 --for-debug-purposes --ref ~/llama.cpp/build-clean-cpu/bin/llama-bench + --ref-flavor clean-cpu --ref-no-affinity`, 16 threads): pp512 1172 vs 927, tg128 88.7 vs 86.0. + Wall clock from launch to terminate: about 90 minutes, of which the build is 12 and the mint 6. +- 2026-09-01 `c8i.4xlarge` (Xeon 6975P-C Granite Rapids, class `x86-amx`), `i-0fb77cb72129e36a2`: + sections 1-7; the TEST gate 65/65 ok with the AMX leg; the class was added on the branch before + launch; the minted `x86-amx.tune-defaults.json` is the shipped one. diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 415578528f..ae6bf27de4 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -27,8 +27,8 @@ winners back. **A change to the sidecar-exchange client (`dasllama/dasllama_exchange.das`), or to a tune-boot path that reaches it, applies `REVIEW_EXCHANGE.md`.** -**Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** A `dasllama/` diff never -opens that checklist on its own. +**Every `dasllama/` change applies this folder's `tests/REVIEW.md` - open it explicitly: the +folder walk does not surface it for a `dasllama/`-only diff.** **A GPU kernel, driver, dispatch-class, or K/V-mirror change - and a GPU kernel A/B race, knockout, or hand-binding arm, wherever the diff puts it - applies `REVIEW_GPU.md`.** @@ -249,3 +249,8 @@ the tuning the profile was meant to save. root) - is a `def` returning it, never a module global with a declaration initializer (`let` or `var`).** A team lane never runs global initializers, so the global reads zero there while every single-threaded run reads the right value. +**A `resize` in `dasllama/` of a buffer whose element count scales with a model dimension is +preceded by a `reserve` of the same count (`reserve_resize` / `grow_resize` / `ensure_length` in +`dasllama_common.das`, or the pair spelled out) - whatever the size looks like at today's +shapes.** A model dimension makes the count unbounded, and a bare grow past the heap's +unreserved-size cap (64 MB) panics the load on the first big model rather than at the call site. diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 64d77f451b..c8cec4edf1 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -37,6 +37,11 @@ family, or backend no existing cell exercises - a q8 or f32 serving lane and a G included. A kernel or form that only makes a path an existing cell already serves faster is not a new capability - it re-mints that cell's row on at least one box instead. +**A diff that makes an already-served path measurably faster re-mints that cell's board row +(`performance/records/`) on at least one box, in the same change.** The board is the module's +public memory of what serving costs; a kernel win that never lands there is invisible to the +next regression check. + **A timing figure PRESENTED AS A MEASUREMENT of a served turn as a whole - tok/s, latency, a whole-turn model or engine comparison - is a defect wherever this module writes it down with no cell behind it: a checked-in doc, a ledger, a code comment, or a PR description.** The diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das new file mode 100644 index 0000000000..c727d4cb70 --- /dev/null +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -0,0 +1,766 @@ +options gen2 +options persistent_heap +options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) +options _dasllama_internal = true + +require daslib/clargs +require daslib/enum_trait +require daslib/jobque_boost +require dasllama/dasllama_par +require dasllama/dasllama_math_gen +require dasllama/dasllama_math +require dasllama/dasllama_kqformat +require dasllama/dasllama_gemm_schema +require llvm/daslib/llvm_env +require math +require strings +require daslib/strings_boost + +//! The kernel microbench - the inner loop of kernel work. Raw kernels, one thread, no jobque, +//! no model; synthetic planes, because the timing is data-independent. GEMV is the decode shape +//! (one token), TILE the prefill batch. Every row of a kernel's `_variants()` registry is timed, +//! the rows of one format interleaved round-robin so frequency drift cancels: under +//! DAS_TUNE_MODE=tune that is the whole [tune_perm] grid, under normal mode the stamped winner +//! alone. Timing only - a spelling's correctness is gen_tune_probe's TEST mode. +//! Run: DAS_TUNE_MODE=tune bin/daslang -jit modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das -- --fmt iq3s +//! The reference row at the same shape: GGML_BENCH_THREADS=1 test-backend-ops perf -o MUL_MAT -p "type_a=iq3_s" +//! (its default MUL_MAT perf case is m=4096 rows, k=14336, n in {1, 512} tokens - this bench's defaults); +//! harness/kernel_ladder.sh runs both sides for every format and joins the table. + +[CommandLineArgs] +struct BenchArgs { + @clarg_short = "f" + @clarg_doc = "KqFmt names, comma-separated (q8,k4,k5,k6,q40,q51,iq4xs,k3,iq3s,iq3xxs,iq4nl,k2,iq2s,iq2xs,iq2xxs; mx4 = mxfp4; q8s16 = q8 over binary16 group scales, the GGUF q8_0 tensor's path), or all" + fmt : string = "all" + @clarg_doc = "Reduction length - the weight row length (k in test-backend-ops terms)" + n : int = 14336 + @clarg_doc = "Rows - output features (m in test-backend-ops terms)" + d : int = 4096 + @clarg_doc = "Tokens for the TILE arm (the prefill batch); 0 skips the arm" + ntok : int = 512 + @clarg_doc = "Timed rounds per row, interleaved across the format's rows; best and median are reported" + rounds : int = 5 + @clarg_doc = "Skip the GEMV (decode) arm" + no_gemv : bool + @clarg_short = "p" + @clarg_doc = "Registry rows to time - comma-separated substrings of the perm suffix, =name for an exact row (default: every row, the reference body included)" + perm : string + @clarg_doc = "TSV rows only, no banner" + tsv : bool + @clarg_doc = "Arena base alignment in bytes (64; 4096 fixes the planes' phase against page boundaries, which the heap otherwise draws per process)" + base_align : int = 64 + @clarg_doc = "Bytes added past the aligned arena base (with --base-align 4096: the planes' phase inside the page)" + base_offset : int = 0 + @clarg_doc = "GEMV the engine's way: the engine's own splitter (matmul_chunks_gemv) and dispatcher (maybe_parallel_for_indexed) over DAS_JOBQUE_THREADS lanes, wall = the slowest lane, the lanes' fastest and slowest summed call time in two extra columns; 0 = one raw call on this thread" + team : bool + @clarg_doc = "Append every round's sample (us, in round order) as one more column - the per-round view of a bimodal row" + each : bool + @clarg_doc = "--team: gemv chunks per lane for the engine's splitter (0 = the engine default, 8; 1 = one equal chunk per lane)" + chunks_per_lane : int = 0 +} + +typedef GemvFn = function<(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void> +typedef TileFn = function<(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void> +typedef GemvQ8Fn = function<(var yp : float?; wp : int8 const?; sp : float const?; xqp : int8 const?; xsp : float const?; n, rb, re : int64) : void> +typedef GemvQ8S16Fn = function<(var yp : float?; wp : int8 const?; sp : uint16 const?; xqp : int8 const?; xsp : float const?; n, rb, re : int64) : void> +typedef TileQ8S16Fn = function<(var yp : float?; wg : int8 const?; sg : uint16 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void> +typedef TileQ8Fn = function<(var yp : float?; wg : int8 const?; sg : float const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void> +typedef GemvMx4Fn = function<(var yp : float?; wn : uint8 const?; we : uint8 const?; xqp : int8 const?; xsp : float const?; n, rb, re : int64) : void> +typedef LayoutFn = function<() : int> + +let TILE_TOKENS = 4l // tokens one kq / q51 / mx4 tile call covers (q8's tokstep companion says its own) +let MX4_NAME = "mx4" // mxfp4 has no KqFmt row; it rides the q8q8 grid as a companion +let S16_NAME = "q8s16" // q8 over binary16 group scales - the wscale_f16 rail a GGUF q8_0 tensor runs (8.5 bits/weight, the reference build's q8_0 bytes) +var g_s16 = false // the current q8-family row is the s16 flavor + +def fmt_of_name(name : string) : KqFmt { + for (f in type) { + if ("{f}" == name) { + return f + } + } + panic("unknown kq format '{name}'") + return KqFmt.q8 +} + +def gemv_variants(f : KqFmt) : array> { + if (f == KqFmt.k4) return <- k4q8_gemv_gen_variants() + if (f == KqFmt.k5) return <- k5q8_gemv_gen_variants() + if (f == KqFmt.k6) return <- k6q8_gemv_gen_variants() + if (f == KqFmt.q40) return <- q40q8_gemv_gen_variants() + if (f == KqFmt.iq4xs) return <- iq4xsq8_gemv_gen_variants() + if (f == KqFmt.k3) return <- k3q8_gemv_gen_variants() + if (f == KqFmt.iq3s) return <- iq3sq8_gemv_gen_variants() + if (f == KqFmt.iq3xxs) return <- iq3xxsq8_gemv_gen_variants() + if (f == KqFmt.iq4nl) return <- iq4nlq8_gemv_gen_variants() + if (f == KqFmt.k2) return <- k2q8_gemv_gen_variants() + if (f == KqFmt.iq2s) return <- iq2sq8_gemv_gen_variants() + if (f == KqFmt.iq2xs) return <- iq2xsq8_gemv_gen_variants() + if (f == KqFmt.iq2xxs) return <- iq2xxsq8_gemv_gen_variants() + if (f == KqFmt.q51) return <- q51q8_gemv_gen_variants() + var none : array> + return <- none +} + +def tile_variants(f : KqFmt) : array> { + if (f == KqFmt.k4) return <- k4q8_tile_gen_variants() + if (f == KqFmt.k5) return <- k5q8_tile_gen_variants() + if (f == KqFmt.k6) return <- k6q8_tile_gen_variants() + if (f == KqFmt.q40) return <- q40q8_tile_gen_variants() + if (f == KqFmt.iq4xs) return <- iq4xsq8_tile_gen_variants() + if (f == KqFmt.k3) return <- k3q8_tile_gen_variants() + if (f == KqFmt.iq3s) return <- iq3sq8_tile_gen_variants() + if (f == KqFmt.iq3xxs) return <- iq3xxsq8_tile_gen_variants() + if (f == KqFmt.iq4nl) return <- iq4nlq8_tile_gen_variants() + if (f == KqFmt.k2) return <- k2q8_tile_gen_variants() + if (f == KqFmt.iq2s) return <- iq2sq8_tile_gen_variants() + if (f == KqFmt.iq2xs) return <- iq2xsq8_tile_gen_variants() + if (f == KqFmt.iq2xxs) return <- iq2xxsq8_tile_gen_variants() + if (f == KqFmt.q51) return <- q51q8_tile_gen_variants() + var none : array> + return <- none +} + +def layout_variants(f : KqFmt) : array> { + if (f == KqFmt.k4) return <- k4q8_layout_gen_variants() + if (f == KqFmt.k5) return <- k5q8_layout_gen_variants() + if (f == KqFmt.k6) return <- k6q8_layout_gen_variants() + if (f == KqFmt.q40) return <- q40q8_layout_gen_variants() + if (f == KqFmt.iq4xs) return <- iq4xsq8_layout_gen_variants() + if (f == KqFmt.k3) return <- k3q8_layout_gen_variants() + if (f == KqFmt.iq3s) return <- iq3sq8_layout_gen_variants() + if (f == KqFmt.iq3xxs) return <- iq3xxsq8_layout_gen_variants() + if (f == KqFmt.iq4nl) return <- iq4nlq8_layout_gen_variants() + if (f == KqFmt.k2) return <- k2q8_layout_gen_variants() + if (f == KqFmt.iq2s) return <- iq2sq8_layout_gen_variants() + if (f == KqFmt.iq2xs) return <- iq2xsq8_layout_gen_variants() + if (f == KqFmt.iq2xxs) return <- iq2xxsq8_layout_gen_variants() + if (f == KqFmt.q51) return <- q51q8_layout_gen_variants() + return <- q8q8_layout_gen_variants() // q8 and its mx4 companion share the q8q8 layout +} + +//! an int-valued companion registry (layout mr, q8's tokstep) keyed by suffix; the companion is +//! stamped in lockstep with its tile, so the row of the same suffix belongs to it +def companion_ints(var lvs : array>) : table { + var vals <- {for (v in lvs); v._0 => int64(invoke(v._1))} + delete lvs + return <- vals +} + +//! a TILE reads either the packed quant plane or a caller-unpacked byte panel (256 bytes per row per superblock: k5, k6 and the five grid formats) - kq_reads_packed_planes is the truth +def tile_kq_bytes_per_sb(f : KqFmt) : int64 => kq_reads_packed_planes(int64(kq_schema_id(f))) ? kq_qsb(f) : 256l + +//! plane bytes per row: kq formats per 256-superblock, q51 per 32-block (20 + 4), q8 n int8 + n/32 f32, mx4 n/2 nibbles + n/32 e8m0 +struct PlaneShape { + gemvW_row : int64 // the gemv's weight plane bytes per row + tileW_row : int64 // the tile's weight plane bytes per row + ws_row : int64 // scale plane bytes per row +} + +def plane_shape(f : KqFmt; mx4 : bool; n : int64) : PlaneShape { + if (mx4) { + return PlaneShape(gemvW_row = n / 2l, tileW_row = n / 2l, ws_row = n / 32l) + } + if (f == KqFmt.q8) { + return PlaneShape(gemvW_row = n, tileW_row = n, ws_row = (n / 32l) * (g_s16 ? 2l : 4l)) + } + if (f == KqFmt.q51) { + return PlaneShape(gemvW_row = (n / 32l) * 20l, tileW_row = (n / 32l) * 20l, ws_row = (n / 32l) * 4l) + } + let nsb = n / 256l + return PlaneShape(gemvW_row = nsb * kq_qsb(f), tileW_row = nsb * tile_kq_bytes_per_sb(f), ws_row = nsb * kq_ssb(f)) +} + +var BASE_ALIGN = 64l // the arena's start alignment - --base-align; the planes' phase against 4 KB pages follows it +var BASE_OFFSET = 0l // --base-offset, added past the aligned start +let ALIGN = 64l // a 64-byte load split across two cache lines is the 2x penalty the fixture must not introduce +let STAGGER = 1088l // 1024 + 64: keeps the six plane starts distinct in their low 12 address bits +let SCALE_BYTE = uint8(0x3C) // as f16 0x3C3C = 1.06, as f32 0x3C3C3C3C = 0.0115, as e8m0 2^-67, as a 6-bit sub-scale 60 - normal in every scale form + +//! every plane of one format in ONE arena at fixed offsets +struct Planes { + arena : array + gemvKq : int64 // the gemv's weight plane + tileKq : int64 // the tile's weight plane (the unpacked panel form where the format has one) + ks : int64 // scale plane + xq : int64 // int8 activations + xs : int64 // f32 activation scales + xbs : int64 // int per-16 activation sums + y : int64 // f32 outputs +} + +//! the next plane start: 64-byte aligned, `slot` staggers past the running end +def next_plane(var end : int64&; bytes : int64; slot : int) : int64 { + let start = ((end + ALIGN - 1l) / ALIGN) * ALIGN + int64(slot) * STAGGER + end = start + bytes + return start +} + +//! the arena's first BASE_ALIGN-aligned byte +def arena_base(var p : Planes) : int64 { + unsafe { + return ((BASE_ALIGN - int64(intptr(addr(p.arena[0])) & uint64(BASE_ALIGN - 1l))) & (BASE_ALIGN - 1l)) + BASE_OFFSET + } +} + +[arch(at = "../../ARCHITECTURE_MEASUREMENT.md#cpu-kernel-bench-fixture")] +def fill_planes(var p : Planes; sh : PlaneShape; n, d, ntok : int64) { + var end = 0l + p.gemvKq = next_plane(end, d * sh.gemvW_row, 0) + p.tileKq = next_plane(end, d * sh.tileW_row, 1) + p.ks = next_plane(end, d * sh.ws_row, 2) + p.xq = next_plane(end, ntok * n, 3) + p.xs = next_plane(end, ntok * n / 32l * 4l, 4) + p.xbs = next_plane(end, ntok * n / 16l * 4l, 5) + p.y = next_plane(end, ntok * d * 4l, 6) + p.arena |> reserve(end + BASE_ALIGN + BASE_OFFSET) + p.arena |> resize(end + BASE_ALIGN + BASE_OFFSET) + let base = arena_base(p) + p.gemvKq += base + p.tileKq += base + p.ks += base + p.xq += base + p.xs += base + p.xbs += base + p.y += base + var seed = 0x1234u + for (i in range64(p.gemvKq, p.gemvKq + d * sh.gemvW_row)) { + seed = seed * 1664525u + 1013904223u + p.arena[i] = uint8(seed >> 24u) + } + for (i in range64(p.tileKq, p.tileKq + d * sh.tileW_row)) { + seed = seed * 1664525u + 1013904223u + p.arena[i] = uint8(seed >> 24u) + } + for (i in range64(p.ks, p.ks + d * sh.ws_row)) { + p.arena[i] = SCALE_BYTE + } + for (i in range64(p.xq, p.xq + ntok * n)) { + seed = seed * 1664525u + 1013904223u + p.arena[i] = uint8(seed >> 24u) + } + unsafe { + var xs = addr(p.arena[p.xs]) + for (i in range64(ntok * n / 32l)) { + xs[i] = 0.01 + } + var xbs = addr(p.arena[p.xbs]) + for (i in range64(ntok * n / 16l)) { + xbs[i] = 128 + } + } +} + +//! one timed row: the samples of every round; best and median are reported +struct RowStat { + perm : string + us : array + lane_lo : int64 // --team: the fastest single lane call seen across the rounds + lane_hi : int64 // --team: the slowest +} + +let TEAM_BURN = 6 // warmup dispatches per row before the team arm times anything +let SOLO_BURN = 3 // the solo arm's unmeasured warm rounds (a core ramps for ~3) +var g_each_round = false // --each + +//! the engine's decode dispatch, verbatim (dasllama_math's matmul shape): the splitter's chunk count over the +//! rows, the team self-serving them; each lane adds its call time to laneP[slot] +def team_gemv(fn : GemvFn; var yp : float?; kqp, ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64; var laneP : int64?) { + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { + let l0 = ref_time_ticks() + invoke(fn, yp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + unsafe { + laneP[slot] += int64(get_time_usec(l0)) + } + } +} + +def team_gemv_q8(fn : GemvQ8Fn; var yp : float?; wp : int8 const?; sp : float const?; xqp : int8 const?; xsp : float const?; n, d : int64; var laneP : int64?) { + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { + let l0 = ref_time_ticks() + invoke(fn, yp, wp, sp, xqp, xsp, n, int64(rb), int64(re)) + unsafe { + laneP[slot] += int64(get_time_usec(l0)) + } + } +} + +def team_gemv_q8s16(fn : GemvQ8S16Fn; var yp : float?; wp : int8 const?; sp : uint16 const?; xqp : int8 const?; xsp : float const?; n, d : int64; var laneP : int64?) { + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { + let l0 = ref_time_ticks() + invoke(fn, yp, wp, sp, xqp, xsp, n, int64(rb), int64(re)) + unsafe { + laneP[slot] += int64(get_time_usec(l0)) + } + } +} + +def team_gemv_mx4(fn : GemvMx4Fn; var yp : float?; wn, we : uint8 const?; xqp : int8 const?; xsp : float const?; n, d : int64; var laneP : int64?) { + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { + let l0 = ref_time_ticks() + invoke(fn, yp, wn, we, xqp, xsp, n, int64(rb), int64(re)) + unsafe { + laneP[slot] += int64(get_time_usec(l0)) + } + } +} + +//! the team arm's interleaved timing: the block dispatches row `row` the engine's way and every lane adds its +//! call time to laneP[slot]; the row keeps the wall (best, median) and the lanes' fastest and slowest sums +[arch(at = "../../ARCHITECTURE_MEASUREMENT.md#cpu-kernel-bench-fixture")] +def time_rows_team(var stats : array; rounds : int; blk : block<(row : int; var laneP : int64?) : void>) { + var laneUs : array + laneUs |> resize(get_total_hw_jobs() + 1) + var laneP = unsafe(addr(laneUs[0])) + for (row in range(length(stats))) { + stats[row].us |> reserve(rounds) + stats[row].lane_lo = 999999999l + for (_w in range(TEAM_BURN)) { + invoke(blk, row, laneP) + } + } + to_log(LOG_INFO, "kq_kernel_bench: timed rounds begin\n") // the profiler sync line, as in time_rows + for (_r in range(rounds)) { + for (row in range(length(stats))) { + for (u in laneUs) { + u = 0l + } + let t0 = ref_time_ticks() + invoke(blk, row, laneP) + stats[row].us |> push(int64(get_time_usec(t0))) + for (u in laneUs) { + if (u > 0l) { + stats[row].lane_lo = min(stats[row].lane_lo, u) + stats[row].lane_hi = max(stats[row].lane_hi, u) + } + } + } + } +} + +def median_us(var us : array) : int64 { + var s := us + sort(s) + let m = length(s) / 2 + let r = length(s) % 2 == 1 ? s[m] : (s[m - 1] + s[m]) / 2l + delete s + return r +} + +def emit_row(var rows : int&; name, tier : string; var st : RowStat; n, d, ntok, nsb : int64; plane_bytes : int64) { + var best = 999999999l + for (u in st.us) { + best = min(best, u) + } + let med = median_us(st.us) + let ns_per_sb = float(best) * 1000.0 / float(d * nsb * ntok) + let gbs = float(plane_bytes) / (float(best) * 1000.0) + let gflops = 2.0 * float(n) * float(d) * float(ntok) / (float(best) * 1000.0) + let mode = g_env_tune.tune_mode + let perm = (st.perm == "reference" && mode != "tune" && mode != "test") ? "stamped" : st.perm // normal mode: the registry's one row IS the stamped winner + let lanes = st.lane_hi > 0l ? "\t{st.lane_lo}\t{st.lane_hi}" : "" + var each = "" + if (g_each_round) { + each = "\t" + join(st.us, ",") + } + print("{name}\t{tier}\t{perm}\t{n}\t{d}\t{ntok}\t{best}\t{med}\t{ns_per_sb}\t{gbs}\t{gflops}{lanes}{each}\n") + rows++ +} + +def perm_wanted(spec, suffix : string) : bool { + if (empty(spec)) { + return true + } + for (tok in split(spec, ",")) { + let tk = strip(tok) + if (empty(tk)) { + continue + } + if (starts_with(tk, "=") ? suffix == slice(tk, 1) : find(suffix, tk) >= 0) { + return true + } + } + return false +} + +//! interleaved timing: round r calls every selected row once before round r + 1 starts +[arch(at = "../../ARCHITECTURE_MEASUREMENT.md#cpu-kernel-bench-fixture")] +def time_rows(var stats : array; rounds : int; blk : block<(row : int) : void>) { + for (row in range(length(stats))) { + stats[row].us |> reserve(rounds) + for (_w in range(SOLO_BURN)) { + invoke(blk, row) + } + } + to_log(LOG_INFO, "kq_kernel_bench: timed rounds begin\n") // external profilers key on this line to skip the JIT startup + for (_r in range(rounds)) { + for (row in range(length(stats))) { + let t0 = ref_time_ticks() + invoke(blk, row) + stats[row].us |> push(int64(get_time_usec(t0))) + } + } +} + +def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; var p : Planes; nsb : int64; sh : PlaneShape) { + let n = int64(cfg.n) + let d = int64(cfg.d) + let plane_bytes = int64(cfg.d) * (sh.gemvW_row + sh.ws_row) + var vs <- gemv_variants(f) + var stats : array + var fns : array + for (v in vs) { + if (perm_wanted(cfg.perm, v._0)) { + stats |> emplace(RowStat(perm = v._0)) + fns |> push(v._1) + } + } + unsafe { + let kqp = addr(p.arena[p.gemvKq]) + let ksp = addr(p.arena[p.ks]) + let xqp = addr(p.arena[p.xq]) + let xsp = addr(p.arena[p.xs]) + let xbsp = addr(p.arena[p.xbs]) + var yp = addr(p.arena[p.y]) + if (cfg.team) { + time_rows_team(stats, cfg.rounds) $(row : int; var laneP : int64?) { + team_gemv(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, d, laneP) + } + } else { + time_rows(stats, cfg.rounds) $(row : int) { + invoke(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) + } + } + if (cfg.tsv) { // the row's answer, one solo call each: modes and seats must agree here before their times are compared + for (fn, st in fns, stats) { + invoke(fn, yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) + var acc = 0.0lf + for (i in range64(d)) { + acc += double(yp[i]) + } + print("# ysum\t{name}\t{st.perm}\t{acc}\t{yp[0]}\t{yp[d - 1l]}\n") + } + } + } + for (st in stats) { + emit_row(rows, name, "gemv", st, n, d, 1l, nsb, plane_bytes) + } + delete stats + delete fns + delete vs +} + +def bench_tile_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; var p : Planes; nsb : int64; sh : PlaneShape) { + let n = int64(cfg.n) + let d = int64(cfg.d) + let ntok = int64(cfg.ntok) + let qrow = sh.tileW_row + let srow = sh.ws_row + let plane_bytes = int64(cfg.d) * (sh.tileW_row + sh.ws_row) + var mrs <- companion_ints(layout_variants(f)) + var vs <- tile_variants(f) + var stats : array + var fns : array + var mrOf : array + for (v in vs) { + if (!perm_wanted(cfg.perm, v._0)) { + continue + } + let mr = mrs?[v._0] ?? (mrs?["reference"] ?? 0l) + if (mr <= 0l || d % mr != 0l) { + to_log(LOG_WARNING, "kq_kernel_bench: {name} tile '{v._0}' has no layout row (mr={mr}) - skipped\n") + continue + } + stats |> emplace(RowStat(perm = v._0)) + fns |> push(v._1) + mrOf |> push(mr) + } + unsafe { + let kqp = addr(p.arena[p.tileKq]) + let ksp = addr(p.arena[p.ks]) + let xqp = addr(p.arena[p.xq]) + let xsp = addr(p.arena[p.xs]) + let xbsp = addr(p.arena[p.xbs]) + var yp = addr(p.arena[p.y]) + time_rows(stats, cfg.rounds) $(row : int) { + let mr = mrOf[row] + for (g in range64(d / mr)) { + var t0k = 0l + while (t0k < ntok) { + invoke(fns[row], yp, kqp + g * mr * qrow, ksp + g * mr * srow, xqp, xsp, xbsp, n, d, g, t0k) + t0k += TILE_TOKENS + } + } + } + } + for (st in stats) { + emit_row(rows, name, "tile", st, n, d, ntok, nsb, plane_bytes) + } + delete stats + delete fns + delete mrOf + delete vs + delete mrs +} + +//! the q8q8 grid's gemv rows (q8 or its mx4 companion) into stats + typed function lists +def q8_gemv_rows(cfg : BenchArgs; mx4 : bool; var stats : array; var fnsQ : array; var fnsM : array; var fnsS : array) { + if (mx4) { + var vs <- mx4q8_gemv_gen_variants() + for (v in vs) { + if (perm_wanted(cfg.perm, v._0)) { + stats |> emplace(RowStat(perm = v._0)) + fnsM |> push(v._1) + } + } + delete vs + } elif (g_s16) { + var vs <- q8q8_gemv_s16_gen_variants() + for (v in vs) { + if (perm_wanted(cfg.perm, v._0)) { + stats |> emplace(RowStat(perm = v._0)) + fnsS |> push(v._1) + } + } + delete vs + } else { + var vs <- q8q8_gemv_gen_variants() + for (v in vs) { + if (perm_wanted(cfg.perm, v._0)) { + stats |> emplace(RowStat(perm = v._0)) + fnsQ |> push(v._1) + } + } + delete vs + } +} + +//! the q8q8 grid's tile rows with each row's mr and tokens-per-call; skips rows whose mr the row count does not divide +def q8_tile_rows(cfg : BenchArgs; mx4 : bool; d : int64; var stats : array; var fnsQ : array; var fnsM : array; var fnsS : array; var mrOf, stepOf : array) { + var mrs <- companion_ints(q8q8_layout_gen_variants()) + var steps <- companion_ints(q8q8_tokstep_gen_variants()) + if (mx4) { + var vs <- mx4q8_tile_gen_variants() + for (v in vs) { + let mr = mrs?[v._0] ?? (mrs?["reference"] ?? 0l) + if (perm_wanted(cfg.perm, v._0) && mr > 0l && d % mr == 0l) { + stats |> emplace(RowStat(perm = v._0)) + fnsM |> push(v._1) + mrOf |> push(mr) + stepOf |> push(TILE_TOKENS) + } + } + delete vs + } elif (g_s16) { + var vs <- q8q8_tile_s16_gen_variants() + for (v in vs) { + let mr = mrs?[v._0] ?? (mrs?["reference"] ?? 0l) + if (perm_wanted(cfg.perm, v._0) && mr > 0l && d % mr == 0l) { + stats |> emplace(RowStat(perm = v._0)) + fnsS |> push(v._1) + mrOf |> push(mr) + stepOf |> push(steps?[v._0] ?? (steps?["reference"] ?? TILE_TOKENS)) + } + } + delete vs + } else { + var vs <- q8q8_tile_gen_variants() + for (v in vs) { + let mr = mrs?[v._0] ?? (mrs?["reference"] ?? 0l) + if (perm_wanted(cfg.perm, v._0) && mr > 0l && d % mr == 0l) { + stats |> emplace(RowStat(perm = v._0)) + fnsQ |> push(v._1) + mrOf |> push(mr) + stepOf |> push(steps?[v._0] ?? (steps?["reference"] ?? TILE_TOKENS)) + } + } + delete vs + } + delete mrs + delete steps +} + +//! q8 (the q8q8 grid) and its mx4 companion: int8 / nibble weight planes with f32 / e8m0 scales +def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes; nsb : int64; sh : PlaneShape) { // nolint:STYLE038 — the q8 family's three flavors (f32, s16, mx4) x two arms: one dispatch ladder over shared planes + let n = int64(cfg.n) + let d = int64(cfg.d) + let ntok = int64(cfg.ntok) + let name = mx4 ? MX4_NAME : (g_s16 ? S16_NAME : "q8") + let plane_bytes = int64(cfg.d) * (sh.gemvW_row + sh.ws_row) + var wvs <- q8q8_family_live_variants() // the family's witness rows: the amx one takes the per-process tile permission its tiles need (SIGILL without it), as the probe does + for (w in wvs) { + invoke(w._1) + } + delete wvs + unsafe { + let kqp = addr(p.arena[p.gemvKq]) + let ksp = addr(p.arena[p.ks]) + let xqp = addr(p.arena[p.xq]) + let xsp = addr(p.arena[p.xs]) + let xbsp = addr(p.arena[p.xbs]) + var yp = addr(p.arena[p.y]) + if (!cfg.no_gemv) { + var stats : array + var fnsQ : array + var fnsM : array + var fnsS : array + q8_gemv_rows(cfg, mx4, stats, fnsQ, fnsM, fnsS) + if (cfg.team) { + time_rows_team(stats, cfg.rounds) $(row : int; var laneP : int64?) { + if (mx4) { + team_gemv_mx4(fnsM[row], yp, kqp, ksp, xqp, xsp, n, d, laneP) + } elif (g_s16) { + team_gemv_q8s16(fnsS[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, d, laneP) + } else { + team_gemv_q8(fnsQ[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, d, laneP) + } + } + } else { + time_rows(stats, cfg.rounds) $(row : int) { + if (mx4) { + invoke(fnsM[row], yp, kqp, ksp, xqp, xsp, n, 0l, d) + } elif (g_s16) { + invoke(fnsS[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, 0l, d) + } else { + invoke(fnsQ[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, 0l, d) + } + } + } + for (st in stats) { + emit_row(rows, name, "gemv", st, n, d, 1l, nsb, plane_bytes) + } + delete stats + delete fnsQ + delete fnsM + delete fnsS + } + if (ntok > 0l) { + var stats : array + var fnsQ : array + var fnsM : array + var fnsS : array + var mrOf : array + var stepOf : array + q8_tile_rows(cfg, mx4, d, stats, fnsQ, fnsM, fnsS, mrOf, stepOf) + time_rows(stats, cfg.rounds) $(row : int) { + let mr = mrOf[row] + for (g in range64(d / mr)) { + var t0k = 0l + while (t0k < ntok) { + if (mx4) { + invoke(fnsM[row], yp, kqp + g * mr * sh.tileW_row, ksp + g * mr * sh.ws_row, xqp, xsp, xbsp, n, d, g, t0k) + } elif (g_s16) { + invoke(fnsS[row], yp, reinterpret(kqp + g * mr * sh.tileW_row), reinterpret(ksp + g * mr * sh.ws_row), xqp, xsp, xbsp, n, d, g, t0k) + } else { + invoke(fnsQ[row], yp, reinterpret(kqp + g * mr * sh.tileW_row), reinterpret(ksp + g * mr * sh.ws_row), xqp, xsp, xbsp, n, d, g, t0k) + } + t0k += stepOf[row] + } + } + } + for (st in stats) { + emit_row(rows, name, "tile", st, n, d, ntok, nsb, plane_bytes) + } + delete stats + delete fnsQ + delete fnsM + delete fnsS + delete mrOf + delete stepOf + } + } +} + +//! the requested formats as (name, KqFmt, is_mx4); "all" = every KqFmt row plus mx4 +def formats_of(spec : string) : array> { + var fs : array> + if (spec == "all") { + for (f in type) { + fs |> emplace(("{f}", f, false)) + } + fs |> emplace((MX4_NAME, KqFmt.q8, true)) + fs |> emplace((S16_NAME, KqFmt.q8, false)) + return <- fs + } + for (tok in split(spec, ",")) { + let name = strip(tok) + if (empty(name)) { + continue + } + if (name == MX4_NAME) { + fs |> emplace((MX4_NAME, KqFmt.q8, true)) + } elif (name == S16_NAME) { + fs |> emplace((S16_NAME, KqFmt.q8, false)) + } else { + fs |> emplace((name, fmt_of_name(name), false)) + } + } + return <- fs +} + +[export] +def main : int { + var cfg = BenchArgs() + let rc = parse_args_with_help(cfg, "kq_kernel_bench") + BASE_ALIGN = int64(cfg.base_align) + BASE_OFFSET = int64(cfg.base_offset) + g_each_round = cfg.each + if (cfg.chunks_per_lane > 0) { + set_gemv_chunks_per_lane(cfg.chunks_per_lane) + } + return rc if (rc >= 0) + if (!jit_enabled()) { + to_log(LOG_ERROR, "run with -jit: the kq generators only fire under the JIT\n") + return 2 + } + if (cfg.n % 256 != 0 || cfg.ntok % int(TILE_TOKENS) != 0) { + to_log(LOG_ERROR, "--n must be a multiple of 256 and --ntok of {TILE_TOKENS}\n") + return 2 + } + var fs <- formats_of(cfg.fmt) + if (!cfg.tsv) { + print("kq_kernel_bench: n={cfg.n} d={cfg.d} ntok={cfg.ntok} rounds={cfg.rounds} team={cfg.team} tune_mode={g_env_tune.tune_mode}\n") + print("fmt\ttier\tperm\tn\td\tntok\tbest_us\tmedian_us\tns_per_sb\tGB/s\tGFLOP/s{cfg.team ? "\tlane_lo\tlane_hi" : ""}\n") + } + var rows = 0 + if (cfg.team) { + with_job_que() { + setup_dasllama_jobque_() + let chunks = matmul_chunks_gemv(cfg.d, 1, int64(cfg.n) * int64(cfg.d)) + print("# team: {get_total_hw_jobs() + 1} lanes, the engine's splitter gives {chunks} chunks of {cfg.d / chunks} rows\n") + rows = run_formats(cfg, fs) + } + } else { + rows = run_formats(cfg, fs) + } + delete fs + if (rows == 0) { + to_log(LOG_ERROR, "kq_kernel_bench: no rows - nothing matched --fmt '{cfg.fmt}' / --perm '{cfg.perm}' or every arm was skipped\n") + return 1 + } + return 0 +} + +//! every selected format's arms; the row count printed +def run_formats(cfg : BenchArgs; fs : array>) : int { + var rows = 0 + let nsb = int64(cfg.n) / 256l + for (fe in fs) { + g_s16 = fe._0 == S16_NAME + let sh = plane_shape(fe._1, fe._2, int64(cfg.n)) + var p : Planes + fill_planes(p, sh, int64(cfg.n), int64(cfg.d), int64(max(cfg.ntok, 1))) + if (fe._2 || fe._1 == KqFmt.q8) { + bench_q8_family(rows, cfg, fe._2, p, nsb, sh) + } else { + if (!cfg.no_gemv) { + bench_gemv_kq(rows, cfg, fe._0, fe._1, p, nsb, sh) + } + if (cfg.ntok > 0) { + bench_tile_kq(rows, cfg, fe._0, fe._1, p, nsb, sh) + } + } + delete p + } + return rows +} diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 2d13a33379..e2640f7f5f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -11,6 +11,7 @@ require llvm/daslib/llvm_jit_code require llvm/daslib/llvm_jit_common require daslib/ast_boost require dasllama/dasllama_gemm_schema +require llvm/daslib/llvm_tune // The perm-parameterized Q8 GEMM tile generator (gemm_generator_plan.md part 3, promoted from // the M2 hand perm): emits the mr-row x 4-token register tile as LLVM IR — the loop nest, @@ -96,6 +97,11 @@ struct private TilePerm { // scratch, block i−1's scale fold interleaved with block i's TMUL ops latch : int = 0 // 1 = raw immediate-tmm amx ops (slice K I1): no per-call // ldtilecfg/tilerelease — the cfg companion owns per-chunk config + grid : string = "" // "vbmi": the grid formats' gemv decodes through the VBMI symbol lattice + // (the row form; the tile and the planes are the plain row's) + pf : int = 0 // software prefetch distance in bytes along the gemv's weight stream (0 = none): + // one llvm.prefetch per weight line, pf bytes ahead - memory-level parallelism at + // many lanes, where the hardware prefetcher alone leaves DRAM bandwidth unused } def private parse_perm(gc : LlvmCodeCtx) : TilePerm { @@ -118,6 +124,10 @@ def private parse_perm(gc : LlvmCodeCtx) : TilePerm { if (pv is tInt) { p.pipe = pv as tInt } let lv = find_arg(gc.ann.arguments, "latch") if (lv is tInt) { p.latch = lv as tInt } + let grv = find_arg(gc.ann.arguments, "grid") + if (grv is tString) { p.grid = grv as tString } + let pfv = find_arg(gc.ann.arguments, "pf") + if (pfv is tInt) { p.pf = pfv as tInt } return p } @@ -134,6 +144,7 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint || (p.bias != 0 && p.bias != 128) || (p.pipe != 0 && p.pipe != 1) || (p.latch != 0 && p.latch != 1) + || (p.grid != "") || (p.pf != 0) || (!g_target_is_x64) || (!g_target_x64_amx) || (!g_target_x64_avx512vnni) // the companions' busd512 lattice + zmm fold || (!g_target_os_linux)) return true @@ -169,12 +180,14 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint || (p.nrsplit != 2 && p.nrsplit != 4) || (p.gkstep != 1 && p.gkstep != 2 && p.gkstep != 4) || (p.mr < 4) || (p.pipe != 0) || (p.latch != 0) + || (p.grid != "" && p.grid != "vbmi") + || (p.pf != 0 && p.pf != 1024 && p.pf != 2048 && p.pf != 4096) || (p.bias != 0 && (p.bias != 128 || p.dotPrim != "vpdpbusd"))) return true let rt = q8q8_repack_type(p.mr, p.bias, kgroup_of(p)) if (rt.interleave != p.mr || !rt.f32_scales) return true if (p.dotPrim == "smmla") { // the i8mm MMA leg: NEON widths, row-quad geometry, same q-reg budget as sdot, and the i8mm tier flag (M1 is dotprod-only) - if ((!g_target_is_aarch64) || (!g_target_arm64_i8mm) || (p.width != 128) + if ((!g_target_is_aarch64) || (!g_target_arm64_i8mm) || (p.width != 128) || (p.grid != "") || (p.pf != 0) || (p.mr % 4 != 0) || (2 * p.mr + p.nrsplit * p.mr / 4 + 2 * p.nrsplit + 2 > 32)) return true let mod = LLVMGetGlobalParent(gc.impl) @@ -192,7 +205,7 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint } if (p.dotPrim == "sdot") { // the NEON leg (M2 shape): 128-bit q-regs, 4-row sdot lanes; mr>=8 budgets the FUSED-acc shape (kq shares one lo+hi acc there — half the mr4 formula's int-acc bank) - if ((!g_target_is_aarch64) || (p.width != 128) + if ((!g_target_is_aarch64) || (p.width != 128) || (p.grid != "") || (p.pf != 0) || (p.mr % 4 != 0) || (p.mr >= 8 ? (p.mr + p.nrsplit * p.mr / 4 + 2 * p.nrsplit + 2 > 32) : (2 * p.mr + p.nrsplit * p.mr / 4 + 2 * p.nrsplit + 2 > 32))) return true @@ -210,6 +223,7 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint if ((p.dotPrim != "maddubs" && p.dotPrim != "vpdpbusd" && p.dotPrim != "vpdpbssd") || (!g_target_is_x64) || (p.width != 256 && p.width != 512) + || (p.grid == "vbmi" && (p.width != 512 || p.dotPrim == "vpdpbssd" || !g_target_x64_avx512vbmi)) // the lattice is zmm VBMI || (p.dotPrim == "vpdpbssd" && p.width != 256)) return true // 512-bit bssd is AVX10 — unproven if (p.width == 256) { if ((p.dotPrim == "maddubs" && !g_target_x64_avx2) @@ -224,8 +238,8 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint let vq = p.mr / rv if ((vq > 2) // the weight bank is 8 dword-groups x vq || (p.nrsplit * vq > 8)) return true // accumulator bank (emit_slice phi arrays) - // vreg budget: float + int acc tiles + x splat/sign scratch + weight/|weight| in flight - return 2 * p.nrsplit * vq + 6 > (p.width == 512 ? 32 : 16) + // vreg budget: float + int acc tiles + x splat/sign scratch + weight/|weight| in flight; EVEX targets have 32 ymm too + return 2 * p.nrsplit * vq + 6 > (p.width == 512 || g_target_x64_avx512bw ? 32 : 16) } // emission context threaded through the helpers: builder/types + the per-function value handles @@ -245,10 +259,23 @@ struct private TileEmit { abs_decl : LLVMOpaqueValue? // x64 sign trick: llvm.abs (VPABSB) abs_ty : LLVMOpaqueType? psign_decl : LLVMOpaqueValue? // x64 width-256 sign apply (VPSIGNB) - iq3s_panel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) - iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) - iq3s_smask_g : LLVMOpaqueValue? // iq3s gemv: the [16 x i32] sign-nibble byte-mask global (iq3xxs shares it) - iq3xxs_ksigns_g : LLVMOpaqueValue? // iq3xxs gemv: the [128 x i32] ksigns_iq2xs global (7-bit index -> sign byte) + gridPanel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) + gridGlobal : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) + sign_code_pm1_g : LLVMOpaqueValue? // the row-group grid decode: ksigns code -> eight +-1 bytes, [128 x i64], parity bit folded in + gridRows : bool // set while the row-group grid decode emits: the column dword read (a panel-form win) reverts to byte loads + vbmi : bool // grid = "vbmi": the grid gemv decodes through the VBMI symbol lattice (row form) + pf : int // software prefetch distance along the weight stream (bytes; 0 = none) + prefetch_decl : LLVMOpaqueValue? // llvm.prefetch.p0 + prefetch_ty : LLVMOpaqueType? + vb_plane : LLVMOpaqueValue? [2] // the grid's compact code planes, [256 x i8] each: the entry's low / high symbols + vb_alpha : LLVMOpaqueValue? // the symbol alphabet as a vpshufb table (16 bytes per lane, [64 x i8]) + vb_ksigns : LLVMOpaqueValue? // ksigns_iq2xs as [128 x i8] - one VPERMI2B turns 7-bit codes into sign bytes, parity in + permi2b_decl : LLVMOpaqueValue? // llvm.x86.avx512.vpermi2var.qi.512 + permi2b_ty : LLVMOpaqueType? + mshift_decl : LLVMOpaqueValue? // llvm.x86.avx512.pmultishift.qb.512 + mshift_ty : LLVMOpaqueType? + shufb_decl : LLVMOpaqueValue? // llvm.x86.avx512.pshuf.b.512 (the lattice's alphabet lookup) + shufb_ty : LLVMOpaqueType? gfn : LLVMOpaqueValue? // the function under emit (block creation for emitted inner loops) gctx : LLVMOpaqueContext? psign_ty : LLVMOpaqueType? @@ -422,6 +449,14 @@ def private bias_acc_init(var te : TileEmit; var bi, xlo, xhi : LLVMOpaqueValue? return LLVMBuildShuffleVector(b, te.types, one, one, mask, "binit{tk}") } +//! llvm.prefetch(read, high locality, data) of the weight plane at byte offset `off` +def private prefetch_wg(var te : TileEmit; var off : LLVMOpaqueValue?) { + let b = te.builder + var p = LLVMBuildGEP2(b, te.types.t_int8, te.wg, off, "", false) // the look-ahead runs past the plane's end by design - inbounds would be poison + var args <- [p, te.types->ConstI32(0ul), te.types->ConstI32(3ul), te.types->ConstI32(1ul)] + LLVMBuildCall2(b, te.prefetch_ty, te.prefetch_decl, args, "") +} + // One 32-k block for tokens [tokBase, tokBase+tokCount): 8*rq shared weight vectors x tokens // lane-dots at full width, then the per-block scale fold — the exact op order of // dot_q8q8_laneq4x4_template, so fast-math contraction sees the same input across emitters. @@ -458,6 +493,9 @@ def private emit_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLV var off = boff == 0 ? wb : LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(boff)), "") wv[kg * rq + qd] = load_vec(te, te.vwi8, te.wg, off, "wv{kg * rq + qd}") uw[kg * rq + qd] = abs_w(te, wv[kg * rq + qd], "uw{kg * rq + qd}") + if (te.pf != 0 && tokCount == 1) { // the gemv's weight stream: the same line, pf bytes ahead + prefetch_wg(te, LLVMBuildAdd(b, off, te.types->ConstI64(uint64(te.pf)), "")) + } } } } @@ -887,6 +925,25 @@ def private madd16_flush(var te : TileEmit; var acc, chain : LLVMOpaqueValue?) : return LLVMBuildAdd(b, acc, quads, "a32") } +//! the rows' sub-scales (k6/k3 int8, k4/k5 unsigned 6-bit) as an i16 pair splat (lanes 2r, 2r + 1 = row r): pmaddwd against a chain widens AND scales in one op +def private scale16_dup(var te : TileEmit; var off : LLVMOpaqueValue?; signed : bool; name : string) : LLVMOpaqueValue? { + let b = te.builder + var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) + var p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, off, "") + var s8 = LLVMBuildLoad2Aligned(b, vri8, p, 1u, "") + var s16 = signed ? LLVMBuildSExt(b, s8, te.vni16, "") : LLVMBuildZExt(b, s8, te.vni16, "") + let dup <- [for (i in range(2 * te.rv)); i / 2] + return LLVMBuildShuffleVector(b, te.types, s16, s16, dup, name) +} + +//! the chain flush with the sub-scale folded in - exact in i32 (|chain| <= 32004, |scale| <= 128, 16 flushes per superblock) +def private madd16_flush_scaled(var te : TileEmit; var acc, chain, s16 : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + var wargs <- [chain, s16] + var quads = LLVMBuildCall2(b, te.maddwd_ty, te.maddwd_decl, wargs, "squads") + return LLVMBuildAdd(b, acc, quads, "a32s") +} + // one row-byte load at BYTE offset, each byte splatted across its row's 4 lanes to // vwi8 (constant shufflevector — the backend picks tbl/pshufb itself; no intrinsic needed) def private load_row_bytes_x4(var te : TileEmit; var base, off : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { @@ -910,8 +967,8 @@ def private load_f16_vec_at(var te : TileEmit; var base, off : LLVMOpaqueValue?; // the backend lowers the pair to a single addp) def private pairwise_add_i32(var te : TileEmit; var av, bv : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { let b = te.builder - let evens <- [0, 2, 4, 6] - let odds <- [1, 3, 5, 7] + let evens <- [for (i in range(te.rv)); 2 * i] + let odds <- [for (i in range(te.rv)); 2 * i + 1] var e = LLVMBuildShuffleVector(b, te.types, av, bv, evens, "") var o = LLVMBuildShuffleVector(b, te.types, av, bv, odds, "") return LLVMBuildAdd(b, e, o, name) @@ -924,9 +981,17 @@ def private splat_lane_i32(var te : TileEmit; var v : LLVMOpaqueValue?; lane : i } // or-in of a masked bit test as the nibble's 0x10: w |= (bytes & mask) != 0 ? 0x10 : 0 -// (pcmpeqb+pand shape after lowering — the k5 high-bit deposit) -def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { +// (pcmpeqb+pand shape after lowering — the k5 high-bit deposit). NEON shifts bytes per lane, so the sdot +// lattice moves bit t (lo) / 4 + t (hi) straight onto bit 4: one ushl / ushr and one and +def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; hi : bool; name : string) : LLVMOpaqueValue? { let b = te.builder + if (te.dotKind == DOT_SDOT) { + let w8 = te.width / 8 + var shifts <- [for (i in range(w8)); LLVMConstInt(te.types.t_int8, uint64(hi ? i % 4 : 4 - i % 4), 0)] + var shv = LLVMConstVector(array_data_ptr(shifts), uint(w8)) + var moved = hi ? LLVMBuildLShr(b, bytes, shv, "") : LLVMBuildShl(b, bytes, shv, "") + return LLVMBuildOr(b, w, LLVMBuildAnd(b, moved, splat_i8w(te, 16), ""), name) + } var hit = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, LLVMBuildAnd(b, bytes, maskv, ""), LLVMConstNull(te.vwi8), "") var sel = LLVMBuildSelect(b, hit, splat_i8w(te, 16), LLVMConstNull(te.vwi8), name) return LLVMBuildOr(b, w, sel, name) @@ -935,6 +1000,7 @@ def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; na // One 256-weight SUPERBLOCK, K-quant grp form (te.kq set): weight vectors unpacked // ONCE per (sub-block, dword-group) and dotted against every token — weight-stationary, // bit-exact vs per-token GEMVs (tokCount=1). kq v2 folds integer sub-scales/bsums against Q8_K-form activations. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { // nolint:STYLE037,STYLE038 — one register-coupled superblock loop nest; the k-quant decode arms share live values let b = te.builder let rq = te.rq @@ -945,6 +1011,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f let k2 = te.kq == 2 let k63 = k6 || k3 // k3 rides k6's scale row and fold; only the quant compose and the offset differ let per16 = k63 || k2 // per-16 scale granularity: split lo/hi accumulators, unsummed bs0/bs1 + let wholeSub = te.kq == 4 || te.kq == 40 || k3 || k2 // the i16 chains run the whole sub-block before one widen (k5: 4 madds, k6: 2) let koff = k3 ? 4 : 32 // the signed-lattice offset the bsum fold removes let q40 = te.kq == 40 let qsb = (te.kq == 4 || q40) ? 128 : (k3 ? 96 : (k2 ? 64 : (te.kqBytes ? 256 : (te.kq == 5 ? 160 : 192)))) @@ -959,6 +1026,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f // x64 dot kinds take the activation dword straight from memory (vpbroadcastd, see kq_dot_mem); NEON keeps chunk loads let memBcast = te.dotKind != DOT_SDOT && te.dotKind != DOT_SMMLA let madd16 = memBcast && te.dotKind == DOT_MADDUBS + let fuseScale = madd16 && (k63 || te.kq == 5) // the sub-scale rides the chain flush (pmaddwd): no i32 multiply, no a0/a1; k4 stays unfused - one flush per sub-block loses more than the fold saves var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) // weight-side superblock scale rows: k4/k5 f16 d/dmin; k6/k3 f16 d only; q40 has none (per-32 d loads live below) var dv : LLVMOpaqueValue? [2] @@ -1026,6 +1094,18 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f p16lo[i] = null p16hi[i] = null } + var s16lo : LLVMOpaqueValue? [2] // fuseScale: this sub-block's two sub-scales as i16 pair splats + var s16hi : LLVMOpaqueValue? [2] + if (fuseScale && k63) { + for (qd in range(rq)) { + s16lo[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * blk * mr + qd * te.rv)), ""), true, "s16{blk}_{qd}a") + s16hi[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), true, "s16{blk}_{qd}b") + } + } elif (fuseScale) { // k5: one 6-bit scale per sub-block, one chain + for (qd in range(rq)) { + s16lo[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), false, "s16{blk}_{qd}") + } + } for (j in range(4)) { for (qd in range(rq)) { // weight vectors unpacked ONCE, dotted against every token below (kqBytes panel loads verbatim instead) @@ -1033,17 +1113,13 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var wlo : LLVMOpaqueValue? var whi : LLVMOpaqueValue? if (k3) { - // k3: qs column (blk/4)*8 + j (lo) / + 4 (hi), 2-bit lane at shift 2*(blk%4); hmask column j / j+4, bit blk - let c0 = (blk / 4) * 8 + j - let sh = (blk % 4) * 2 - var q3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(c0 * 4 * mr + qd * w8)), ""), "q3l{blk}_{j * rq + qd}") - var q3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((c0 + 4) * 4 * mr + qd * w8)), ""), "q3h{blk}_{j * rq + qd}") - var h3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + j * 4) * mr + qd * w8)), ""), "h3l{blk}_{j * rq + qd}") - var h3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + (j + 4) * 4) * mr + qd * w8)), ""), "h3h{blk}_{j * rq + qd}") - var lo2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3l, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") - var hi2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3h, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") - var lob = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3l, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") - var hib = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3h, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") + var q3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 2) * 4 * mr + qd * w8)), ""), "q3l{blk}_{qd}") + var q3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 2 + 1) * 4 * mr + qd * w8)), ""), "q3h{blk}_{qd}") + var h3 = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + blk * 4) * mr + qd * w8)), ""), "h3{blk}_{qd}") + var lo2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3l, splat_i8w(te, 2 * j), ""), splat_i8w(te, 3), "") + var hi2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3h, splat_i8w(te, 2 * j), ""), splat_i8w(te, 3), "") + var lob = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3, splat_i8w(te, j), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") + var hib = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3, splat_i8w(te, 4 + j), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") wlo = LLVMBuildOr(b, lo2, lob, "wlo3{blk}_{j * rq + qd}") whi = LLVMBuildOr(b, hi2, hib, "whi3{blk}_{j * rq + qd}") } elif (k2) { @@ -1066,17 +1142,16 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f if (te.kq == 5 && !te.kqBytes) { var hoff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(128 * mr + (blk * 4 + j) * mr + qd * te.rv)), "") var hb = load_row_bytes_x4(te, te.wg, hoff, "qh{blk}_{j * rq + qd}") - wlo = or_bit_x10(te, wlo, hb, maskLo, "wlo5{blk}_{j * rq + qd}") - whi = or_bit_x10(te, whi, hb, maskHi, "whi5{blk}_{j * rq + qd}") + wlo = or_bit_x10(te, wlo, hb, maskLo, false, "wlo5{blk}_{j * rq + qd}") + whi = or_bit_x10(te, whi, hb, maskHi, true, "whi5{blk}_{j * rq + qd}") } elif (k6 && !te.kqBytes) { - let g2 = (blk % 4) * 2 - let hbase = 128 * mr + ((blk / 4) * 8) * 4 * mr - var hloOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + j * 4 * mr + qd * w8)), "") - var hhiOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + (j + 4) * 4 * mr + qd * w8)), "") - var qhlo = load_vec(te, te.vwi8, te.wg, hloOff, "qhl{blk / 4}_{j * rq + qd}") - var qhhi = load_vec(te, te.vwi8, te.wg, hhiOff, "qhh{blk / 4}_{j * rq + qd}") - var dlo = LLVMBuildAnd(b, LLVMBuildLShr(b, qhlo, splat_i8w(te, g2), ""), splat_i8w(te, 3), "") - var dhi = LLVMBuildAnd(b, LLVMBuildLShr(b, qhhi, splat_i8w(te, g2), ""), splat_i8w(te, 3), "") + let hbase = 128 * mr + blk * 2 * 4 * mr + var hloOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + qd * w8)), "") + var hhiOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + 4 * mr + qd * w8)), "") + var qhlo = load_vec(te, te.vwi8, te.wg, hloOff, "qhl{blk}_{qd}") + var qhhi = load_vec(te, te.vwi8, te.wg, hhiOff, "qhh{blk}_{qd}") + var dlo = LLVMBuildAnd(b, LLVMBuildLShr(b, qhlo, splat_i8w(te, 2 * j), ""), splat_i8w(te, 3), "") + var dhi = LLVMBuildAnd(b, LLVMBuildLShr(b, qhhi, splat_i8w(te, 2 * j), ""), splat_i8w(te, 3), "") wlo = LLVMBuildOr(b, wlo, LLVMBuildShl(b, dlo, splat_i8w(te, 4), ""), "wlo6{blk}_{j * rq + qd}") whi = LLVMBuildOr(b, whi, LLVMBuildShl(b, dhi, splat_i8w(te, 4), ""), "whi6{blk}_{j * rq + qd}") } @@ -1110,21 +1185,41 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } } - // i16 chain flushes at the format's overflow bound: k5 after 4 madds, k6 (and k3, conservatively) after 2; k4/q40 run the full 8 - if (madd16 && (j == 1 || j == 3) && te.kq != 4 && !q40) { + // i16 chain flushes at the format's overflow bound: k5 after 4 madds, k6 after 2 (2 x 16002); wholeSub formats run the sub-block + if (madd16 && (j == 1 || j == 3) && !wholeSub) { for (k in range(tokCount * rq)) { - a0[k] = madd16_flush(te, a0[k], p16lo[k]) + if (fuseScale) { + iacc[k] = madd16_flush_scaled(te, iacc[k], p16lo[k], s16lo[k % rq]) + if (per16) { + iacc[k] = madd16_flush_scaled(te, iacc[k], p16hi[k], s16hi[k % rq]) + p16hi[k] = null + } + } else { + a0[k] = madd16_flush(te, a0[k], p16lo[k]) + if (per16) { + a1[k] = madd16_flush(te, a1[k], p16hi[k]) + p16hi[k] = null + } + } p16lo[k] = null + } + } + } + if (madd16 && wholeSub) { + for (k in range(tokCount * rq)) { + if (fuseScale) { + iacc[k] = madd16_flush_scaled(te, iacc[k], p16lo[k], s16lo[k % rq]) + if (per16) { + iacc[k] = madd16_flush_scaled(te, iacc[k], p16hi[k], s16hi[k % rq]) + p16hi[k] = null + } + } else { + a0[k] = madd16_flush(te, a0[k], p16lo[k]) if (per16) { a1[k] = madd16_flush(te, a1[k], p16hi[k]) p16hi[k] = null } } - } - } - if (madd16 && (te.kq == 4 || q40)) { // one widen per (token, qd) for the whole sub-block - for (k in range(tokCount * rq)) { - a0[k] = madd16_flush(te, a0[k], p16lo[k]) p16lo[k] = null } } @@ -1190,9 +1285,11 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var bs0v = splat_i32(te, bs0, "bs0v{tk}_{blk}") var bs1v = splat_i32(te, bs1, "bs1v{tk}_{blk}") for (qd in range(rq)) { - var ia = LLVMBuildAdd(b, LLVMBuildMul(b, scv[qd], a0[i * rq + qd], ""), LLVMBuildMul(b, mnv[qd], a1[i * rq + qd], ""), "ia6{tk}_{blk}_{qd}") + if (!fuseScale) { // the fused flushes already scaled the dots into iacc + var ia = LLVMBuildAdd(b, LLVMBuildMul(b, scv[qd], a0[i * rq + qd], ""), LLVMBuildMul(b, mnv[qd], a1[i * rq + qd], ""), "ia6{tk}_{blk}_{qd}") + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tk}_{blk}_{qd}") + } var ba = LLVMBuildAdd(b, LLVMBuildMul(b, scv[qd], bs0v, ""), LLVMBuildMul(b, mnv[qd], bs1v, ""), "ba6{tk}_{blk}_{qd}") - iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tk}_{blk}_{qd}") bacc[i * rq + qd] = LLVMBuildAdd(b, bacc[i * rq + qd], ba, "ba{tk}_{blk}_{qd}") } } elif (k2) { @@ -1207,8 +1304,10 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } else { for (qd in range(rq)) { - var acc = fuseAcc ? a0[i * rq + qd] : LLVMBuildAdd(b, a0[i * rq + qd], a1[i * rq + qd], "acc{tk}_{blk}_{qd}") - iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], LLVMBuildMul(b, scv[qd], acc, ""), "ia{tk}_{blk}_{qd}") + if (!fuseScale) { // the fused flushes already scaled the dots into iacc + var acc = fuseAcc ? a0[i * rq + qd] : LLVMBuildAdd(b, a0[i * rq + qd], a1[i * rq + qd], "acc{tk}_{blk}_{qd}") + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], LLVMBuildMul(b, scv[qd], acc, ""), "ia{tk}_{blk}_{qd}") + } if (!vecBsums) { bacc[i * rq + qd] = LLVMBuildAdd(b, bacc[i * rq + qd], LLVMBuildMul(b, mnv[qd], a32v, ""), "ba{tk}_{blk}_{qd}") } @@ -1253,9 +1352,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } -// The iq3s gemv gather globals: the grid as a [512 x i32] private constant, the 4-bit sign -// nibble expanded to a byte mask via a [16 x i32] table (negation is (w ^ m) + (m & 0x01010101); -// grid magnitudes are odd 1..15, so no carry crosses a byte lane). +// The iq3s gemv gather global: the grid as a [512 x i32] private constant. def private iq3s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq3s.grid") if (grid == null) { @@ -1271,84 +1368,270 @@ def private iq3s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + te.gridGlobal = grid +} + +// The five grid formats' per-dword decode - the raw grid word of the row whose packed grp region +// starts at byte `base`, for block `blk`, sign group `l`, half `h`; loads shared across (blk, l, h) fold in LLVM's CSE. +def private wg_byte(var te : TileEmit; var base : LLVMOpaqueValue?; off : int) : LLVMOpaqueValue? { + var p = LLVMBuildGEP2(te.builder, te.types.t_int8, te.wg, LLVMBuildAdd(te.builder, base, te.types->ConstI64(uint64(off)), ""), "") + return LLVMBuildZExt(te.builder, LLVMBuildLoad2Aligned(te.builder, te.types.t_int8, p, 1u, ""), te.types.t_int32, "") +} + +def private wg_i32(var te : TileEmit; var base : LLVMOpaqueValue?; off : int) : LLVMOpaqueValue? { + var p = LLVMBuildGEP2(te.builder, te.types.t_int8, te.wg, LLVMBuildAdd(te.builder, base, te.types->ConstI64(uint64(off)), ""), "") + return LLVMBuildLoad2Aligned(te.builder, te.types.t_int32, p, 4u, "") +} + +//! a packed-plane byte position to its grp column offset (4-byte columns, mr rows each) +def private wg_col(mr, pos : int) : int => (pos / 4) * mr * 4 + pos % 4 + +//! the byte at packed position `pos`, read as one dword load of its column with the byte split off in +//! registers - the four positions of a column share the load through CSE +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#grid-decode-forms")] +def private wg_col_byte(var te : TileEmit; var base : LLVMOpaqueValue?; pos : int) : LLVMOpaqueValue? { + if (te.gridRows) return wg_byte(te, base, wg_col(te.interleave, pos)) + let b = te.builder + var col = wg_i32(te, base, (pos / 4) * te.interleave * 4) + let sh = 8 * (pos % 4) + var v = sh == 0 ? col : LLVMBuildLShr(b, col, LLVMConstInt(te.types.t_int32, uint64(sh), 0), "") + return sh == 24 ? v : LLVMBuildAnd(b, v, LLVMConstInt(te.types.t_int32, 255ul, 0), "") +} + +//! grid[gidx] - a raw magnitude word; the block emitter applies the signs per vector +def private grid_word(var te : TileEmit; var gidx : LLVMOpaqueValue?) : LLVMOpaqueValue? { + var gp = LLVMBuildGEP2(te.builder, te.types.t_int32, te.gridGlobal, gidx, "") + return LLVMBuildLoad2Aligned(te.builder, te.types.t_int32, gp, 4u, "") +} + +//! the u64 grid's word pair at even index `idx2`: one 8-byte load, half `h` split off in registers +//! (the h = 0 and h = 1 decodes share the load through CSE) +def private grid_pair_half(var te : TileEmit; var idx2 : LLVMOpaqueValue?; h : int) : LLVMOpaqueValue? { + let b = te.builder + var g64 = grid_pair_i64(te, idx2) + return LLVMBuildTrunc(b, h == 0 ? g64 : LLVMBuildLShr(b, g64, LLVMConstInt(te.types.t_int64, 32ul, 0), ""), te.types.t_int32, "") +} + +//! the u64 grid's word pair at even index `idx2` as one i64 - a row's 8 weights in one load +def private grid_pair_i64(var te : TileEmit; var idx2 : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + var gp = LLVMBuildGEP2(b, te.types.t_int64, te.gridGlobal, LLVMBuildLShr(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), ""), "") + return LLVMBuildLoad2Aligned(b, te.types.t_int64, gp, 4u, "") +} + +//! iq3s: 8-bit qs byte + the block's qh bit = the 9-bit grid index +def private decode_iq3s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + let b = te.builder + let mr = te.interleave + var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) + var qv = wg_byte(te, base, wg_col(mr, blk * 8 + 2 * l + h)) + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - h - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 256ul, 0), "") + return grid_word(te, LLVMBuildOr(b, qv, hb, "")) +} + +//! iq3xxs: an 8-bit index into the halved grid, read through its column dword +def private decode_iq3xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + return grid_word(te, wg_col_byte(te, base, blk * 8 + 2 * l + h)) +} + +//! iq2xs: row `base`'s u16 word for (blk, l) - low 9 bits the grid index, the top 7 the ksigns code, +//! read as its two column bytes +def private iq2xs_word16(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l : int) : LLVMOpaqueValue? { + let b = te.builder + let lp = (blk * 4 + l) * 2 + var qlo = wg_col_byte(te, base, lp) + var qhi = wg_col_byte(te, base, lp + 1) + return LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "") +} + +//! the iq2 formats' doubled index into the u64 grid's word pairs for row `base`, (blk, l): iq2s the qs byte +//! under the qh 0x300 window, iq2xs the u16 word's low 9 bits, iq2xxs block b's grid byte (column 2b) +def private iq2_idx2(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l : int) : LLVMOpaqueValue? { + let b = te.builder + var idx : LLVMOpaqueValue? + if (te.kq == 23) { + let mr = te.interleave + var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 0x300ul, 0), "") + idx = LLVMBuildOr(b, wg_byte(te, base, wg_col(mr, blk * 4 + l)), hb, "") + } elif (te.kq == 24) { + idx = LLVMBuildAnd(b, iq2xs_word16(te, base, blk, l), LLVMConstInt(te.types.t_int32, 511ul, 0), "") + } else { + idx = wg_byte(te, base, 2 * blk * te.interleave * 4 + l) + } + return LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") +} + +//! row `base`'s 7-bit ksigns code for (blk, l) - iq2xs the u16 word's top bits, iq2xxs/iq3xxs the aux32's 7-bit fields +def private kq_row_sign_code(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l : int) : LLVMOpaqueValue? { + let b = te.builder + if (te.kq == 24) return LLVMBuildLShr(b, iq2xs_word16(te, base, blk, l), LLVMConstInt(te.types.t_int32, 9ul, 0), "") + var aux = wg_i32(te, base, (te.kq == 34 ? 16 + blk : 2 * blk + 1) * te.interleave * 4) + var sh = l == 0 ? aux : LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), "") + return LLVMBuildAnd(b, sh, LLVMConstInt(te.types.t_int32, 127ul, 0), "") +} + +//! the eight +-1 bytes of a ksigns code: one table row (bit k of the expanded byte set = lane k negative) +def private sign_pm1_row(var te : TileEmit; var code : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + var gp = LLVMBuildGEP2(b, te.types.t_int64, te.sign_code_pm1_g, code, "") + return LLVMBuildBitCast(b, LLVMBuildLoad2Aligned(b, te.types.t_int64, gp, 8u, ""), LLVMVectorType(te.types.t_int8, 8u), "") +} + +//! rows r0.. of a row group's +-1 sign bytes off the plane's sign column (iq3s/iq2s): byte r*4 + l +//! broadcast over the row's 8 lanes, bit k selects lane k, the mask | 1 +def private sign_col_pm1(var te : TileEmit; var col : LLVMOpaqueValue?; r0, l : int; name : string) : LLVMOpaqueValue? { + let b = te.builder + let w8 = te.width / 8 + let idx <- [for (i in range(w8)); (r0 + i / 8) * 4 + l] + var bytes = LLVMBuildShuffleVector(b, te.types, col, col, idx, "sgp") + var bits <- [for (i in range(w8)); LLVMConstInt(te.types.t_int8, uint64(1 << (i % 8)), 0)] + var bitsv = LLVMConstVector(array_data_ptr(bits), uint(w8)) + var m = LLVMBuildSExt(b, LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntEQ, LLVMBuildAnd(b, bytes, bitsv, ""), bitsv, ""), te.vwi8, "sgpm") + return LLVMBuildOr(b, m, splat_i8w(te, 1), name) +} + +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#grid-decode-forms")] +def private grid_rows_path(te : TileEmit) : bool { + if (te.vbmi) return true + if (te.dotKind == DOT_SDOT) return te.rv == 4 + if (te.dotKind != DOT_MADDUBS && te.dotKind != DOT_VPDPBUSD && te.dotKind != DOT_VPDPBSSD) return false + let cls = tune_cpu_class() + if (cls == "x86-vnni512") return te.kq == 25 + if (cls == "x86-amx") return te.kq == 25 || te.kq == 34 + return false +} + +//! eight +-1 bytes for sign byte `sb`, lane k negative when bit k is set +def public pm1_of(sb : int) : uint64 { // test_kquant round-trips the sign expansion + var v = 0ul + for (k in range(8)) { + v |= uint64(((sb >> k) & 1) != 0 ? 0xFF : 0x01) << uint64(8 * k) + } + return v +} + +//! the row-pair decode's sign table: the 128 ksigns codes with the code's parity as the 8th bit +//! (ksigns[i] == i | parity(i) << 7), each as its eight +-1 bytes +def private sign_pm1_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + if (!grid_rows_path(te)) return // only the row-group grid decode reads it + te.sign_code_pm1_g = LLVMGetNamedGlobal(mod, "dasllama.sign.code_pm1") + if (te.sign_code_pm1_g != null) return + var codes : array + codes |> reserve(128) + for (c in range(128)) { + var par = c ^ (c >> 4) + par ^= par >> 2 + par ^= par >> 1 + codes |> push(LLVMConstInt(te.types.t_int64, pm1_of(c | ((par & 1) << 7)), 0)) + } + te.sign_code_pm1_g = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int64, 128u), "dasllama.sign.code_pm1") + LLVMSetInitializer(te.sign_code_pm1_g, LLVMConstArray(te.types.t_int64, array_data_ptr(codes), 128u)) + LLVMSetGlobalConstant(te.sign_code_pm1_g, 1) + LLVMSetLinkage(te.sign_code_pm1_g, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(te.sign_code_pm1_g, 64u) +} + +//! block `blk`'s sign bytes for rows qd*rv.., byte (r, l) at r*4 + l - the plane's own column for +//! iq3s/iq2s, synthesized from the 7-bit ksigns codes for the other three (the 8th bit is the +//! code's parity, ksigns[i] == i | parity(i) << 7); one masked negate per weight vector applies them +def private kq_sign_bytes(var te : TileEmit; var wbP : LLVMOpaqueValue?; blk, qd : int; name : string) : LLVMOpaqueValue? { + let b = te.builder + let mr = te.interleave + let w8 = te.width / 8 + let colBytes = uint64(mr * 4) + if (te.kq == 33 || te.kq == 23) { // the plane carries the sign bytes + let col = (te.kq == 33 ? 18 : 8) + blk + return load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(col) * colBytes + uint64(qd * w8)), ""), name) + } + var codesCol : LLVMOpaqueValue? [2] // iq2xs: columns 2blk (l = 0, 1) and 2blk+1 (l = 2, 3) of u16 words; else one aux32 column + if (te.kq == 24) { + for (c in range(2)) { + codesCol[c] = load_vec(te, te.vni32, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(2 * blk + c) * colBytes + uint64(qd * w8)), ""), "sgw{blk}_{c}") } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) + } else { + let col = te.kq == 34 ? 16 + blk : 2 * blk + 1 + codesCol[0] = load_vec(te, te.vni32, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(col) * colBytes + uint64(qd * w8)), ""), "sga{blk}") + } + var bytes = LLVMConstNull(te.vni32) + for (l in range(4)) { + var code : LLVMOpaqueValue? + if (te.kq == 24) { + var word = l % 2 == 0 ? LLVMBuildAnd(b, codesCol[l / 2], splat_i32n(te, 65535), "") : LLVMBuildLShr(b, codesCol[l / 2], splat_i32n(te, 16), "") + code = LLVMBuildLShr(b, word, splat_i32n(te, 9), "") + } else { + code = LLVMBuildAnd(b, l == 0 ? codesCol[0] : LLVMBuildLShr(b, codesCol[0], splat_i32n(te, 7 * l), ""), splat_i32n(te, 127), "") + } + var par = LLVMBuildXor(b, code, LLVMBuildLShr(b, code, splat_i32n(te, 4), ""), "") + par = LLVMBuildXor(b, par, LLVMBuildLShr(b, par, splat_i32n(te, 2), ""), "") + par = LLVMBuildXor(b, par, LLVMBuildLShr(b, par, splat_i32n(te, 1), ""), "") + var sb = LLVMBuildOr(b, code, LLVMBuildShl(b, LLVMBuildAnd(b, par, splat_i32n(te, 1), ""), splat_i32n(te, 7), ""), "") + bytes = LLVMBuildOr(b, bytes, l == 0 ? sb : LLVMBuildShl(b, sb, splat_i32n(te, 8 * l), ""), "") } - te.iq3s_smask_g = smask + return LLVMBuildBitCast(b, bytes, te.vwi8, name) +} + +//! one masked negate per weight vector: byte (r, k) of the mask is 0xFF when bit (h*4 + k) of row r's +//! sign byte for group `l` is set - a constant shuffle broadcasts each row's byte to its four lanes +def private apply_sign_col(var te : TileEmit; var w, col : LLVMOpaqueValue?; l, h : int; name : string) : LLVMOpaqueValue? { + let b = te.builder + let w8 = te.width / 8 + let idx <- [for (i in range(w8)); (i / 4) * 4 + l] + var bytes = LLVMBuildShuffleVector(b, te.types, col, col, idx, "sgb") + var bits : array + bits |> reserve(w8) + for (i in range(w8)) { + bits |> push(LLVMConstInt(te.types.t_int8, uint64(1 << (h * 4 + i % 4)), 0)) + } + var bitsv = LLVMConstVector(array_data_ptr(bits), uint(w8)) + var m = LLVMBuildSExt(b, LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntEQ, LLVMBuildAnd(b, bytes, bitsv, ""), bitsv, ""), te.vwi8, "sgm") + return LLVMBuildSub(b, LLVMBuildXor(b, w, m, ""), m, name) +} + +def private decode_kq_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + if (te.kq == 33) return decode_iq3s_dword(te, base, blk, l, h) + if (te.kq == 34) return decode_iq3xxs_dword(te, base, blk, l, h) + return grid_pair_half(te, iq2_idx2(te, base, blk, l), h) } -// iq3s gemv: gather superblock `sbi`'s grid words + signs off the PACKED grp planes into the -// alloca panel — one emitted loop over the mr rows (32 grid words unrolled per iteration); a -// single straight-line loop block, so the phi's back edge is the block itself. -def private emit_iq3s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { +//! bytes per row per superblock of a panel format's packed grp region +def private kq_gather_stride(kq : int) : int => kq == 33 ? 104 : (kq == 34 ? 96 : (kq == 23 ? 72 : 64)) + +//! byte offset of the (blk, l, h) dword group in the panel: two 32-group halves, mr dwords per group +def private panel_group_off(mr, blk, l, h : int) : int { + let e = l * 8 + h * 4 + return (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 +} + +// gemv, gather="panel": superblock `sbi` off the PACKED grp planes into the alloca panel - one +// emitted loop over the mr rows (64 decoded dwords per iteration), a single straight-line loop +// block, so the phi's back edge is the block itself. +def private emit_kq_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { let b = te.builder let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 104)), "iq3s.wb") + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr) * uint64(kq_gather_stride(te.kq))), "kq.wb") var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3s.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3s.cont") + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "kq.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "kq.cont") LLVMBuildBr(b, head) LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq3s.r") + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "kq.r") var preVals <- [te.types->ConstI64(0ul)] var preBlocks <- [pre] LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq3s.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq3s.base") - let c256 = LLVMConstInt(te.types.t_int32, 256ul, 0) - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "kq.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "kq.base") for (blk in range(8)) { - let hp = 64 + blk - var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((hp / 4) * mr * 4 + hp % 4)), ""), "") - var qh = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "iq3s.qh{blk}") for (l in range(4)) { - let sp = 72 + blk * 4 + l - var sgp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((sp / 4) * mr * 4 + sp % 4)), ""), "") - var sgn = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, sgp, 1u, ""), te.types.t_int32, "iq3s.sg{blk}_{l}") - var iv : LLVMOpaqueValue? [2] - for (h in range(2)) { - let qp1 = blk * 8 + 2 * l + h - var qpp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((qp1 / 4) * mr * 4 + qp1 % 4)), ""), "") - var qv = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qpp, 1u, ""), te.types.t_int32, "") - var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - h - 2 * l), 0), ""), c256, "") - iv[h] = LLVMBuildOr(b, qv, hb, "iq3s.i{blk}_{l}_{h}") - } - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") for (h in range(2)) { - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, iv[h], "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq3s.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq3s.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + var w = decode_kq_dword(te, base, blk, l, h) + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.gridPanel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(panel_group_off(mr, blk, l, h))), ""), "") LLVMBuildStore(b, w, dp) } } } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq3s.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq3s.more") + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "kq.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "kq.more") LLVMBuildCondBr(b, more, head, cont) var loopVals <- [rNext] var loopBlocks <- [head] @@ -1356,9 +1639,7 @@ def private emit_iq3s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { LLVMPositionBuilderAtEnd(b, cont) } -// The iq3xxs gemv gather globals: the HALVED grid as a [256 x i32] private constant, ksigns -// as a [128 x i32] (7-bit index -> sign byte, bit 7 = parity), plus the shared smask table -// (the same nibble -> byte-mask expansion; magnitudes 2..31 are nonzero, so no carry). +// The iq3xxs gemv gather global: the HALVED grid as a [256 x i32] private constant. def private iq3xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.grid") if (grid == null) { @@ -1374,106 +1655,11 @@ def private iq3xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid - var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") - if (ksg == null) { - let kt = ksigns_iq2xs() - var elems3 : array - elems3 |> reserve(128) - for (i in range(128)) { - elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) - } - ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") - LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) - LLVMSetGlobalConstant(ksg, 1) - LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(ksg, 16u) - } - te.iq3xxs_ksigns_g = ksg - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) - } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) - } - te.iq3s_smask_g = smask + te.gridGlobal = grid } -// iq3xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the -// iq3s gather's shape with the aux32 sign word (ONE aligned i32 column load per block) and -// ksigns lookups in place of the explicit sign bytes, and no 9th grid-index bit. -def private emit_iq3xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { - let b = te.builder - let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 96)), "iq3x.wb") - var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3x.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3x.cont") - LLVMBuildBr(b, head) - LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq3x.r") - var preVals <- [te.types->ConstI64(0ul)] - var preBlocks <- [pre] - LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq3x.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq3x.base") - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let c127 = LLVMConstInt(te.types.t_int32, 127ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) - for (blk in range(8)) { - var axp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((16 + blk) * mr * 4)), ""), "") - var aux = LLVMBuildLoad2Aligned(b, te.types.t_int32, axp, 4u, "iq3x.aux{blk}") - for (l in range(4)) { - var sidx = LLVMBuildAnd(b, l == 0 ? aux : LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), c127, "") - var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") - var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq3x.sg{blk}_{l}") - var iv : LLVMOpaqueValue? [2] - for (h in range(2)) { - let qp1 = blk * 8 + 2 * l + h - var qpp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((qp1 / 4) * mr * 4 + qp1 % 4)), ""), "") - iv[h] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qpp, 1u, ""), te.types.t_int32, "iq3x.i{blk}_{l}_{h}") - } - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") - for (h in range(2)) { - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, iv[h], "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq3x.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq3x.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") - LLVMBuildStore(b, w, dp) - } - } - } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq3x.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq3x.more") - LLVMBuildCondBr(b, more, head, cont) - var loopVals <- [rNext] - var loopBlocks <- [head] - LLVMAddIncoming(rPhi, loopVals, loopBlocks) - LLVMPositionBuilderAtEnd(b, cont) -} -// The iq2s gemv gather globals: the u64 grid as a [2048 x i32] private constant (1024 entries -// as low/high word pairs), plus the shared smask table (magnitudes {8, 25, 43} are nonzero - -// no carry crosses a byte lane on negation). +// The iq2s gemv gather global: the u64 grid as a [2048 x i32] private constant (low/high word pairs). def private iq2s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2s.grid") if (grid == null) { @@ -1489,92 +1675,11 @@ def private iq2s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) - } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) - } - te.iq3s_smask_g = smask + te.gridGlobal = grid } -// iq2s gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the -// iq3s gather's shape with the 10-bit index (qs byte | qh's 0x300 window) doubled into the -// u64 grid's low/high word pair; the sign byte is explicit like iq3s's. -def private emit_iq2s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { - let b = te.builder - let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 72)), "iq2s.wb") - var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2s.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2s.cont") - LLVMBuildBr(b, head) - LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2s.r") - var preVals <- [te.types->ConstI64(0ul)] - var preBlocks <- [pre] - LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2s.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq2s.base") - let c768 = LLVMConstInt(te.types.t_int32, 0x300ul, 0) - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) - for (blk in range(8)) { - let hp = 64 + blk - var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((hp / 4) * mr * 4 + hp % 4)), ""), "") - var qh = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "iq2s.qh{blk}") - for (l in range(4)) { - let sp = 32 + blk * 4 + l - var sgp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((sp / 4) * mr * 4 + sp % 4)), ""), "") - var sgn = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, sgp, 1u, ""), te.types.t_int32, "iq2s.sg{blk}_{l}") - let ip = blk * 4 + l - var ipp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((ip / 4) * mr * 4 + ip % 4)), ""), "") - var iv0 = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, ipp, 1u, ""), te.types.t_int32, "") - var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), c768, "") - var idx = LLVMBuildOr(b, iv0, hb, "iq2s.i{blk}_{l}") - var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") - for (h in range(2)) { - var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2s.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2s.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") - LLVMBuildStore(b, w, dp) - } - } - } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2s.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2s.more") - LLVMBuildCondBr(b, more, head, cont) - var loopVals <- [rNext] - var loopBlocks <- [head] - LLVMAddIncoming(rPhi, loopVals, loopBlocks) - LLVMPositionBuilderAtEnd(b, cont) -} -// The iq2xs gemv gather globals: the u64 grid as a [1024 x i32] private constant (512 entries -// as low/high word pairs), the shared ksigns table, and the shared smask. +// The iq2xs gemv gather global: the u64 grid as a [1024 x i32] private constant (low/high word pairs). def private iq2xs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2xs.grid") if (grid == null) { @@ -1590,46 +1695,10 @@ def private iq2xs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid - var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") - if (ksg == null) { - let kt = ksigns_iq2xs() - var elems3 : array - elems3 |> reserve(128) - for (i in range(128)) { - elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) - } - ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") - LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) - LLVMSetGlobalConstant(ksg, 1) - LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(ksg, 16u) - } - te.iq3xxs_ksigns_g = ksg - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) - } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) - } - te.iq3s_smask_g = smask + te.gridGlobal = grid } -// The iq2xxs gemv gather globals: the u64 grid as a [512 x i32] private constant (256 entries -// as low/high word pairs), the shared ksigns table, and the shared smask. +// The iq2xxs gemv gather global: the u64 grid as a [512 x i32] private constant (low/high word pairs). def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2xxs.grid") if (grid == null) { @@ -1645,164 +1714,439 @@ def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid - var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") - if (ksg == null) { - let kt = ksigns_iq2xs() - var elems3 : array - elems3 |> reserve(128) - for (i in range(128)) { - elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) - } - ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") - LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) - LLVMSetGlobalConstant(ksg, 1) - LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(ksg, 16u) - } - te.iq3xxs_ksigns_g = ksg - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) + te.gridGlobal = grid +} + + + +//! the 8 activation bytes at byte offset `off` of token 0, replicated over every 64-bit lane of the +//! weight width (ld1r after lowering): the dot operand for a [row 8w | row 8w ...] weight vector +def private bcast_qword_x(var te : TileEmit; var off : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { + let b = te.builder + let n = te.width / 64 + var p = LLVMBuildGEP2(b, te.types.t_int8, te.x[0], off, "") + var q = LLVMBuildLoad2Aligned(b, te.types.t_int64, p, 1u, "") + var vn = LLVMVectorType(te.types.t_int64, uint(n)) + var v = LLVMBuildInsertElement(b, LLVMGetUndef(vn), q, LLVMConstInt(te.types.t_int32, 0ul, 0), "") + let zeros <- [for (_i in range(n)); 0] + var spl = LLVMBuildShuffleVector(b, te.types, v, v, zeros, "") + return LLVMBuildBitCast(b, spl, te.vwi8, name) +} + +//! `rows` <8 x i8> vectors side by side as one weight-width vector (pairwise shuffles - ins after lowering) +def private concat_rows(var te : TileEmit; var rows : array; lanes : int = 8) : LLVMOpaqueValue? { + if (length(rows) == 1) return rows[0] + let b = te.builder + let idx <- [for (i in range(2 * lanes)); i] + var nxt <- [for (i in range(length(rows) / 2)); LLVMBuildShuffleVector(b, te.types, rows[2 * i], rows[2 * i + 1], idx, "")] + return concat_rows(te, nxt, 2 * lanes) +} + +//! one row group's dot with its +-1 sign bytes: the signed-weight lattices (sdot, vpdpbssd) multiply them +//! into the magnitudes; the unsigned-weight ones (maddubs, vpdpbusd) put them on the activation copy +def private grid_rows_dot(var te : TileEmit; var acc, w, sg, xv : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + if (te.dotKind == DOT_SDOT || te.dotKind == DOT_VPDPBSSD) { + var args <- [acc, LLVMBuildMul(b, w, sg, "ws"), xv] + return LLVMBuildCall2(b, te.dotKind == DOT_SDOT ? te.sdot_ty : te.dp_ty, te.dotKind == DOT_SDOT ? te.sdot_decl : te.dp_decl, args, "dot") + } + var xs : LLVMOpaqueValue? + if (te.width == 256) { + var pargs <- [xv, sg] + xs = LLVMBuildCall2(b, te.psign_ty, te.psign_decl, pargs, "signx") + } else { + var neg = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, sg, LLVMConstNull(te.vwi8), "sneg") + xs = LLVMBuildSelect(b, neg, LLVMBuildSub(b, LLVMConstNull(te.vwi8), xv, ""), xv, "signx") + } + if (te.dotKind == DOT_VPDPBUSD) { + var dargs <- [acc, w, xs] + return LLVMBuildCall2(b, te.dp_ty, te.dp_decl, dargs, "dot") + } + var margs <- [w, xs] + var pairs = LLVMBuildCall2(b, te.madd_ty, te.madd_decl, margs, "pairs") + var wargs <- [pairs, ones_i16(te)] + return LLVMBuildAdd(b, acc, LLVMBuildCall2(b, te.maddwd_ty, te.maddwd_decl, wargs, "quads"), "dot") +} + +//! row `base`'s 8 raw grid weights for (blk, l) as <8 x i8>: the iq2 formats' u64 grid entry in one +//! load, iq3s/iq3xxs the two dword decodes side by side +def private row_weights8(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l : int) : LLVMOpaqueValue? { + let b = te.builder + var v8i8 = LLVMVectorType(te.types.t_int8, 8u) + if (te.kq != 33 && te.kq != 34) return LLVMBuildBitCast(b, grid_pair_i64(te, iq2_idx2(te, base, blk, l)), v8i8, "") + var w0 = decode_kq_dword(te, base, blk, l, 0) + var w1 = decode_kq_dword(te, base, blk, l, 1) + var v2 = LLVMVectorType(te.types.t_int32, 2u) + var v = LLVMBuildInsertElement(b, LLVMGetUndef(v2), w0, LLVMConstInt(te.types.t_int32, 0ul, 0), "") + v = LLVMBuildInsertElement(b, v, w1, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + return LLVMBuildBitCast(b, v, v8i8, "") +} + +//! the lattice's constant tables, emitted once per module: the grid's two compact code planes (entry e's +//! low / high half - four 2-bit symbols for the iq2 alphabet, two 3-bit symbols for iq3), the alphabet as a +//! per-lane vpshufb table, and ksigns_iq2xs whole (128 bytes: exactly the two registers one VPERMI2B indexes) +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#vbmi-lattice")] +def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + let tag = vbmi_tag(te.kq) + te.vb_plane[0] = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.p0") + te.vb_plane[1] = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.p1") + te.vb_alpha = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.alpha") + te.vb_ksigns = LLVMGetNamedGlobal(mod, "dasllama.vbmi.ksigns") + if (te.vb_plane[0] != null && te.vb_plane[1] != null && te.vb_alpha != null && te.vb_ksigns != null) return + let alpha <- vbmi_alphabet(te.kq) + let entries = vbmi_entries(te.kq) + var p0 : array + var p1 : array + p0 |> reserve(entries) + p1 |> reserve(entries) + if (te.kq == 34) { + vbmi_planes_iq3xxs(te, p0, p1, alpha) + } elif (te.kq == 33) { + vbmi_planes_iq3s(te, p0, p1, alpha) + } elif (te.kq == 25) { + vbmi_planes_iq2xxs(te, p0, p1, alpha) + } elif (te.kq == 24) { + vbmi_planes_iq2xs(te, p0, p1, alpha) + } else { + vbmi_planes_iq2s(te, p0, p1, alpha) + } + te.vb_plane[0] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p0", p0) + te.vb_plane[1] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p1", p1) + var al : array + al |> reserve(64) + for (k in range(64)) { + al |> push(LLVMConstInt(te.types.t_int8, uint64(k % 16 < length(alpha) ? alpha[k % 16] : 0), 0)) + } + te.vb_alpha = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.alpha", al) + if (te.vb_ksigns == null) { + let ks = ksigns_iq2xs() + var kv : array + kv |> reserve(128) + for (k in range(128)) { + kv |> push(LLVMConstInt(te.types.t_int8, uint64(ks[k]), 0)) } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) + te.vb_ksigns = vbmi_const_bytes(te, mod, "dasllama.vbmi.ksigns", kv) + } +} + +//! an iq3 entry is one word of 4 weights: weights 0-1 and 2-3 per plane +def private vbmi_planes_iq3xxs(te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq3xxs_grid() + for (e in range(256)) { + vbmi_push_iq3(te, p0, p1, g[e], alpha) + } +} + +def private vbmi_planes_iq3s(te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq3s_grid() + for (e in range(512)) { + vbmi_push_iq3(te, p0, p1, g[e], alpha) } - te.iq3s_smask_g = smask } -// iq2xs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the -// iq2s gather's two-word u64 form with the u16 qs word's 9-bit index and its high-7-bit -// KSIGNS lookup in place of explicit sign bytes (the iq3xxs sign path). -def private emit_iq2xs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { +//! an iq2 entry is two words: weights 0-3, 4-7 - one plane each +def private vbmi_planes_iq2xxs(te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq2xxs_grid2() + for (e in range(256)) { + vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) + } +} + +def private vbmi_planes_iq2xs(te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq2xs_grid2() + for (e in range(512)) { + vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) + } +} + +def private vbmi_planes_iq2s(te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq2s_grid2() + for (e in range(1024)) { + vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) + } +} + +def private vbmi_push_iq3(te : TileEmit; var p0, p1 : array; word : uint; alpha : array) { + p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(word, alpha, 3, 2)), 0)) + p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(word >> 16u, alpha, 3, 2)), 0)) +} + +def private vbmi_push_iq2(te : TileEmit; var p0, p1 : array; w0, w1 : uint; alpha : array) { + p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(w0, alpha, 2, 4)), 0)) + p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(w1, alpha, 2, 4)), 0)) +} + +//! the grid's symbol alphabet: the halved iq3xxs magnitudes, iq3s's odd magnitudes, the iq2 family's three +def public vbmi_alphabet(kq : int) : array { // test_kquant round-trips the code planes + if (kq == 34) return <- [2, 6, 10, 14, 18, 22, 26, 31] + if (kq == 33) return <- [1, 3, 5, 7, 9, 11, 13, 15] + return <- [8, 25, 43] +} + +//! the grid gemv's module-level tables: the format's grid, the ksigns +-1 table when the row form reads it, the lattice's +def private grid_emit_globals(var te : TileEmit; fmt : int; mod : LLVMOpaqueModule?) { + if (fmt == 33) { + iq3s_emit_globals(te, mod) + } elif (fmt == 23) { + iq2s_emit_globals(te, mod) + } elif (fmt == 24) { + iq2xs_emit_globals(te, mod) + } elif (fmt == 25) { + iq2xxs_emit_globals(te, mod) + } else { + iq3xxs_emit_globals(te, mod) + } + sign_pm1_emit_globals(te, mod) + if (te.vbmi) { + vbmi_emit_globals(te, mod) + } +} + +def private vbmi_tag(kq : int) : string => kq == 25 ? "iq2xxs" : (kq == 24 ? "iq2xs" : (kq == 23 ? "iq2s" : (kq == 34 ? "iq3xxs" : "iq3s"))) + +//! grid entries per format: the compact plane's byte count +def private vbmi_entries(kq : int) : int => (kq == 25 || kq == 34) ? 256 : (kq == 23 ? 1024 : 512) + +//! a grid word's low `nbytes` weight bytes as packed symbols (weight t at bits t*bits); a byte outside the alphabet is a broken table +def public vbmi_pack_word(word : uint; alpha : array; bits, nbytes : int) : int { + var code = 0 + for (t in range(nbytes)) { + let wb = int((word >> uint(8 * t)) & 0xFFu) + let si = alpha |> find_index(wb) + if (si < 0) { + panic("vbmi lattice: grid byte {wb} is outside the alphabet") + } + code |= si << (bits * t) + } + return code +} + +def private vbmi_const_bytes(te : TileEmit; mod : LLVMOpaqueModule?; name : string; var elems : array) : LLVMOpaqueValue? { + var g = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int8, uint(length(elems))), name) + LLVMSetInitializer(g, LLVMConstArray(te.types.t_int8, array_data_ptr(elems), uint(length(elems)))) + LLVMSetGlobalConstant(g, 1) + LLVMSetLinkage(g, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(g, 64u) + return g +} + +//! one 64-byte slice of a constant byte table (the lattice's register operands; loop-invariant, LLVM hoists them) +def private vbmi_table_part(var te : TileEmit; var tbl : LLVMOpaqueValue?; part : int; name : string) : LLVMOpaqueValue? { let b = te.builder - let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 64)), "iq2x.wb") - var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2x.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2x.cont") - LLVMBuildBr(b, head) - LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2x.r") - var preVals <- [te.types->ConstI64(0ul)] - var preBlocks <- [pre] - LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2x.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq2x.base") - let c511 = LLVMConstInt(te.types.t_int32, 511ul, 0) - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) - for (blk in range(8)) { + var p = LLVMBuildGEP2(b, te.types.t_int8, tbl, te.types->ConstI64(uint64(part * 64)), "") + return LLVMBuildLoad2Aligned(b, te.vwi8, p, 64u, name) +} + +def private vbmi_permi2b(var te : TileEmit; var lo, idx, hi : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { + var args <- [lo, idx, hi] + return LLVMBuildCall2(te.builder, te.permi2b_ty, te.permi2b_decl, args, name) +} + +//! VPMULTISHIFTQB: byte j of each qword = the 8 bits of the source qword starting at bit ctrl[j] +def private vbmi_mshift(var te : TileEmit; var v : LLVMOpaqueValue?; ctrl : array; name : string) : LLVMOpaqueValue? { + var cs : array + cs |> reserve(64) + for (k in range(64)) { + cs |> push(LLVMConstInt(te.types.t_int8, uint64(ctrl[k % 8]), 0)) + } + var args <- [LLVMConstVector(array_data_ptr(cs), 64u), v] + return LLVMBuildCall2(te.builder, te.mshift_ty, te.mshift_decl, args, name) +} + +//! 64 indices through a byte table of `entries` (256 / 512 / 1024): VPERMI2B covers 128 entries per pair of +//! table registers, index bit 7 blends the pairs, and the caller's masks (the index's 9th and 10th bits, per +//! lane) blend the 256-entry halves above that +def private vbmi_lookup(var te : TileEmit; var tbl : LLVMOpaqueValue?; entries : int; var idx : LLVMOpaqueValue?; var masks : array; name : string) : LLVMOpaqueValue? { // nolint:LINT014 — var for non-const value handles: a const element cannot feed LLVMBuildSelect + let b = te.builder + var level <- [for (k in range(entries / 128)); vbmi_permi2b(te, vbmi_table_part(te, tbl, 2 * k, "{name}.t{2 * k}"), idx, vbmi_table_part(te, tbl, 2 * k + 1, "{name}.t{2 * k + 1}"), "{name}.l{k}")] + var high = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, idx, LLVMConstNull(te.vwi8), "{name}.b7") + var depth = 0 + while (length(level) > 1) { + var sel = depth == 0 ? high : masks[depth - 1] + var nxt <- [for (j in range(length(level) / 2)); LLVMBuildSelect(b, sel, level[2 * j + 1], level[2 * j], "{name}.s{depth}_{j}")] + level |> clear() + level |> push_from(nxt) + depth++ + } + return level[0] +} + +//! the unsigned-weight dot with the signs as a byte mask on the activation copy: x' = (x ^ m) - m +def private rows_dot_masked(var te : TileEmit; var acc, w, m, xv : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + var xs = LLVMBuildSub(b, LLVMBuildXor(b, xv, m, ""), m, "signx") + if (te.dotKind == DOT_VPDPBUSD) { + var dargs <- [acc, w, xs] + return LLVMBuildCall2(b, te.dp_ty, te.dp_decl, dargs, "dot") + } + var margs <- [w, xs] + var pairs = LLVMBuildCall2(b, te.madd_ty, te.madd_decl, margs, "pairs") + var wargs <- [pairs, ones_i16(te)] + return LLVMBuildAdd(b, acc, LLVMBuildCall2(b, te.maddwd_ty, te.maddwd_decl, wargs, "quads"), "dot") +} + +//! the block's 64-byte column `col` for rows qd*rv.. (row r's four bytes at r*4) +def private vbmi_col(var te : TileEmit; var wbP : LLVMOpaqueValue?; col, qd : int; name : string) : LLVMOpaqueValue? { + let off = uint64(col * te.interleave * 4 + qd * (te.width / 8)) + return load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(te.builder, wbP, te.types->ConstI64(off), ""), name) +} + +//! the row's qh byte for this block (packed byte position 64 + blk: column 16 + blk/4, byte blk%4) broadcast +//! over its four lanes, then bit `bit0 + stride * (lane % 4)` as a lane mask - the iq3s / iq2s index bits above the qs byte +def private vbmi_qh_mask(var te : TileEmit; var qhCol : LLVMOpaqueValue?; blk, bit0, stride : int; name : string) : LLVMOpaqueValue? { + let b = te.builder + let w8 = te.width / 8 + let idx <- [for (k in range(w8)); (k / 4) * 4 + blk % 4] + var bytes = LLVMBuildShuffleVector(b, te.types, qhCol, qhCol, idx, "{name}.q") + var bits <- [for (k in range(w8)); LLVMConstInt(te.types.t_int8, uint64(1 << (bit0 + stride * (k % 4))), 0)] + var bitsv = LLVMConstVector(array_data_ptr(bits), uint(w8)) + return LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, LLVMBuildAnd(b, bytes, bitsv, ""), LLVMConstNull(te.vwi8), name) +} + +//! the row's sign bytes from its four 7-bit ksigns codes (lane r*4+l), parity included: one VPERMI2B over the table +def private vbmi_ksigns(var te : TileEmit; var codes : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { + var ks0 = vbmi_table_part(te, te.vb_ksigns, 0, "ks0") + var ks1 = vbmi_table_part(te, te.vb_ksigns, 1, "ks1") + return vbmi_permi2b(te, ks0, LLVMBuildAnd(te.builder, codes, splat_i8w(te, 127), ""), ks1, name) +} + +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#vbmi-lattice")] +def private emit_grid_rows_blk_vbmi(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { // nolint:STYLE037,STYLE038 — one block's decode: the five formats' index assembly and the shared expansion + let b = te.builder + let n = te.width / 64 + let w8 = te.width / 8 + let iq3 = te.kq == 33 || te.kq == 34 + let entries = vbmi_entries(te.kq) + var alpha = LLVMBuildLoad2Aligned(b, te.vwi8, te.vb_alpha, 64u, "alpha") + for (qd in range(te.rq)) { + let ncol = iq3 ? 2 : 1 + var plo : LLVMOpaqueValue? [2] + var phi : LLVMOpaqueValue? [2] + var sgn : LLVMOpaqueValue? + for (c in range(ncol)) { + var idx : LLVMOpaqueValue? + var masks : array + if (te.kq == 25 || te.kq == 34) { + idx = vbmi_col(te, wbP, 2 * blk + c, qd, "gi{blk}_{c}") + } elif (te.kq == 33) { + idx = vbmi_col(te, wbP, 2 * blk + c, qd, "gi{blk}_{c}") + var qh = vbmi_col(te, wbP, 16 + blk / 4, qd, "qh{blk}") + masks |> push(vbmi_qh_mask(te, qh, blk, 4 * c, 1, "qm{blk}_{c}")) + } elif (te.kq == 23) { + idx = vbmi_col(te, wbP, blk, qd, "gi{blk}") + var qh = vbmi_col(te, wbP, 16 + blk / 4, qd, "qh{blk}") + masks |> push(vbmi_qh_mask(te, qh, blk, 0, 2, "qm{blk}_0")) + masks |> push(vbmi_qh_mask(te, qh, blk, 1, 2, "qm{blk}_1")) + sgn = vbmi_col(te, wbP, 8 + blk, qd, "sg{blk}") + } else { // 24: u16 words - columns 2blk (l 0, 1) and 2blk+1 (l 2, 3), lane r*4+l takes word l's low then high byte + var ca = vbmi_col(te, wbP, 2 * blk, qd, "wa{blk}") + var cb = vbmi_col(te, wbP, 2 * blk + 1, qd, "wb{blk}") + let lo <- [for (k in range(w8)); (k % 4 < 2 ? 0 : w8) + (k / 4) * 4 + 2 * (k % 2)] + let hi <- [for (k in range(w8)); (k % 4 < 2 ? 0 : w8) + (k / 4) * 4 + 2 * (k % 2) + 1] + idx = LLVMBuildShuffleVector(b, te.types, ca, cb, lo, "gi{blk}") + var hib = LLVMBuildShuffleVector(b, te.types, ca, cb, hi, "gh{blk}") + masks |> push(LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, LLVMBuildAnd(b, hib, splat_i8w(te, 1), ""), LLVMConstNull(te.vwi8), "qm{blk}")) + sgn = vbmi_ksigns(te, LLVMBuildLShr(b, hib, splat_i8w(te, 1), ""), "sg{blk}") + } + plo[c] = vbmi_lookup(te, te.vb_plane[0], entries, idx, masks, "plo{blk}_{c}") + phi[c] = vbmi_lookup(te, te.vb_plane[1], entries, idx, masks, "phi{blk}_{c}") + } + if (te.kq == 33) { + sgn = vbmi_col(te, wbP, 18 + blk, qd, "sg{blk}") + } elif (te.kq == 25 || te.kq == 34) { // the aux32's four 7-bit codes per row, two rows per qword + var aux = vbmi_col(te, wbP, te.kq == 34 ? 16 + blk : 2 * blk + 1, qd, "aux{blk}") + sgn = vbmi_ksigns(te, vbmi_mshift(te, aux, [0, 7, 14, 21, 32, 39, 46, 53], "sc{blk}"), "sg{blk}") + } + var acc : LLVMOpaqueValue? [2] + var acc1 : LLVMOpaqueValue? [2] + for (g in range(2)) { + acc[g] = LLVMConstNull(te.vni32) + acc1[g] = LLVMConstNull(te.vni32) + } for (l in range(4)) { - let lp = (blk * 4 + l) * 2 - var qlp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((lp / 4) * mr * 4 + lp % 4)), ""), "") - var qlo = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qlp, 1u, ""), te.types.t_int32, "") - var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((lp / 4) * mr * 4 + lp % 4 + 1)), ""), "") - var qhi = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "") - var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "iq2x.w16{blk}_{l}") - var idx = LLVMBuildAnd(b, w16, c511, "iq2x.i{blk}_{l}") - var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var sidx = LLVMBuildLShr(b, w16, LLVMConstInt(te.types.t_int32, 9ul, 0), "") - var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") - var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq2x.sg{blk}_{l}") - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") - for (h in range(2)) { - var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2x.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2x.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") - LLVMBuildStore(b, w, dp) + var xv = bcast_qword_x(te, LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32 + l * 8)), ""), "x8{blk}_{l}") + for (g in range(2)) { + let r0 = g * n + var q : LLVMOpaqueValue? + if (!iq3) { // qword i <- row r0+i's low-plane byte, high-plane byte (the rest don't care) + let sel <- [for (k in range(w8)); (k % 8 == 1 ? w8 : 0) + (r0 + k / 8) * 4 + l] + q = LLVMBuildShuffleVector(b, te.types, plo[0], phi[0], sel, "wq{blk}_{l}_{g}") + q = LLVMBuildAnd(b, vbmi_mshift(te, q, [0, 2, 4, 6, 8, 10, 12, 14], "ws{blk}_{l}_{g}"), splat_i8w(te, 3), "") + } else { // qword i <- the two entries (h = 0, 1) of row r0+i, low then high plane each + let c = l / 2 + let p0 = (2 * l) % 4 + let sel <- [for (k in range(w8)); (k % 8 == 1 || k % 8 == 3 ? w8 : 0) + (r0 + k / 8) * 4 + p0 + (k % 8 == 2 || k % 8 == 3 ? 1 : 0)] + q = LLVMBuildShuffleVector(b, te.types, plo[c], phi[c], sel, "wq{blk}_{l}_{g}") + q = LLVMBuildAnd(b, vbmi_mshift(te, q, [0, 3, 8, 11, 16, 19, 24, 27], "ws{blk}_{l}_{g}"), splat_i8w(te, 7), "") + } + var sargs <- [alpha, q] + var w = LLVMBuildCall2(b, te.shufb_ty, te.shufb_decl, sargs, "w{blk}_{l}_{g}") + let ssel <- [for (k in range(w8)); (r0 + k / 8) * 4 + l] + var sq = LLVMBuildShuffleVector(b, te.types, sgn, sgn, ssel, "sq{blk}_{l}_{g}") + sq = LLVMBuildAnd(b, vbmi_mshift(te, sq, [0, 1, 2, 3, 4, 5, 6, 7], "sb{blk}_{l}_{g}"), splat_i8w(te, 1), "") + var m = LLVMBuildSub(b, LLVMConstNull(te.vwi8), sq, "sm{blk}_{l}_{g}") + let hi = iq2 && l >= 2 + var d = rows_dot_masked(te, hi ? acc1[g] : acc[g], w, m, xv) + if (hi) { + acc1[g] = d + } else { + acc[g] = d + } } } + a[qd] = LLVMBuildAdd(b, a[qd], pairwise_add_i32(te, acc[0], acc[1], "ap{blk}_{qd}"), "") + if (iq2) { + a1[qd] = LLVMBuildAdd(b, a1[qd], pairwise_add_i32(te, acc1[0], acc1[1], "ap1{blk}_{qd}"), "") + } } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2x.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2x.more") - LLVMBuildCondBr(b, more, head, cont) - var loopVals <- [rNext] - var loopBlocks <- [head] - LLVMAddIncoming(rPhi, loopVals, loopBlocks) - LLVMPositionBuilderAtEnd(b, cont) } -// iq2xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - block -// b's grid BYTES (column 2b) double into the u64 grid's word pair, its aux32 (column 2b+1) -// carries the 7-bit KSIGNS indices (the per-32 scale nibble rides the scale row, not the gather). -def private emit_iq2xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { +//! One block of a grid format's decode as ROW GROUPS: a weight-width vector is width/64 rows x 8 weights +//! straight from the grid words (one u64 grid entry per iq2 row), signed by +-1 bytes - the ksigns table row +//! per code or the plane's sign column (iq3s/iq2s); one pairwise add per group pair yields the fold's per-row lanes. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#grid-decode-forms")] +def private emit_grid_rows_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var signCol : LLVMOpaqueValue? [2]; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { let b = te.builder - let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 64)), "iq2xx.wb") - var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2xx.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2xx.cont") - LLVMBuildBr(b, head) - LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2xx.r") - var preVals <- [te.types->ConstI64(0ul)] - var preBlocks <- [pre] - LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2xx.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq2xx.base") - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let c127 = LLVMConstInt(te.types.t_int32, 127ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) - for (blk in range(8)) { - var a0p = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((2 * blk + 1) * mr * 4)), ""), "") - var aux = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, a0p, 1u, ""), te.types.t_int32, "") - for (k in range(1, 4)) { - var akp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((2 * blk + 1) * mr * 4 + k)), ""), "") - var ab = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, akp, 1u, ""), te.types.t_int32, "") - aux = LLVMBuildOr(b, aux, LLVMBuildShl(b, ab, LLVMConstInt(te.types.t_int32, uint64(8 * k), 0), ""), "iq2xx.aux{blk}_{k}") + let n = te.width / 64 + let colSigns = te.kq == 33 || te.kq == 23 || te.dotKind != DOT_SDOT // x86: the sign column's mask beats four table loads + three inserts per group; the sdot lattice keeps the table for the ksigns formats + for (qd in range(te.rq)) { + var acc : LLVMOpaqueValue? [2] // per row group; the lo strip (l < 2) when iq2 + var acc1 : LLVMOpaqueValue? [2] // iq2: the hi strip (l >= 2) + for (g in range(2)) { + acc[g] = LLVMConstNull(te.vni32) + acc1[g] = LLVMConstNull(te.vni32) } for (l in range(4)) { - var ip = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64(2 * blk * mr * 4 + l)), ""), "") - var idx = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, ip, 1u, ""), te.types.t_int32, "iq2xx.i{blk}_{l}") - var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var sidx = LLVMBuildAnd(b, LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), c127, "") - var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") - var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq2xx.sg{blk}_{l}") - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") - for (h in range(2)) { - var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2xx.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2xx.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") - LLVMBuildStore(b, w, dp) + var xv = bcast_qword_x(te, LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32 + l * 8)), ""), "x8{blk}_{l}") + for (g in range(2)) { + let r0 = g * n + var rows : array + var sgs : array + rows |> reserve(n) + sgs |> reserve(n) + for (i in range(n)) { + var base = LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64((qd * te.rv + r0 + i) * 4)), "") + rows |> push(row_weights8(te, base, blk, l)) + if (!colSigns) { + sgs |> push(sign_pm1_row(te, kq_row_sign_code(te, base, blk, l))) + } + } + var w = concat_rows(te, rows) + var sg = colSigns ? sign_col_pm1(te, signCol[qd], r0, l, "sp{blk}_{l}_{qd}_{g}") : concat_rows(te, sgs) + let hi = iq2 && l >= 2 + var d = grid_rows_dot(te, hi ? acc1[g] : acc[g], w, sg, xv) + if (hi) { + acc1[g] = d + } else { + acc[g] = d + } } } + a[qd] = LLVMBuildAdd(b, a[qd], pairwise_add_i32(te, acc[0], acc[1], "ap{blk}_{qd}"), "") + if (iq2) { + a1[qd] = LLVMBuildAdd(b, a1[qd], pairwise_add_i32(te, acc1[0], acc1[1], "ap1{blk}_{qd}"), "") + } } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2xx.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2xx.more") - LLVMBuildCondBr(b, more, head, cont) - var loopVals <- [rNext] - var loopBlocks <- [head] - LLVMAddIncoming(rPhi, loopVals, loopBlocks) - LLVMPositionBuilderAtEnd(b, cont) } // One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) @@ -1816,22 +2160,18 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let panel = te.kq == 33 || te.kq == 34 || te.kq == 23 || te.kq == 24 || te.kq == 25 // iq3s/iq3xxs/iq2s/iq2xs/iq2xxs: the byte-expanded signed panel replaces the nibble+LUT decode let iq2 = te.kq == 23 || te.kq == 24 // iq2s/iq2xs: per-16 UNSIGNED strips - lo/hi halves keep split accumulators let nl = te.kq == 45 // iq4nl: the 44 LUT decode with q40's PER-BLOCK f16 d fold (no sub-scales, no min term) - let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first + let gather = panel && te.gridPanel != null // gemv mode: gather this superblock off the packed planes first + // the decode shape composes row GROUPS straight from the grid words (emit_grid_rows_blk) - no panel + let pairs = gather && tokCount == 1 && grid_rows_path(te) var wbase = te.wg var wb : LLVMOpaqueValue? + var wbP : LLVMOpaqueValue? if (gather) { - if (te.kq == 33) { - emit_iq3s_gather(te, sbi) - } elif (te.kq == 23) { - emit_iq2s_gather(te, sbi) - } elif (te.kq == 24) { - emit_iq2xs_gather(te, sbi) - } elif (te.kq == 25) { - emit_iq2xxs_gather(te, sbi) - } else { - emit_iq3xxs_gather(te, sbi) + wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr) * uint64(kq_gather_stride(te.kq))), "kq.wb") + if (!pairs) { + emit_kq_gather(te, sbi) } - wbase = te.iq3s_panel + wbase = te.gridPanel wb = te.types->ConstI64(0ul) } else { wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (panel ? 256 : 128))), "wb") @@ -1872,7 +2212,20 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var a[i] = LLVMConstNull(te.vni32) a1[i] = LLVMConstNull(te.vni32) } - for (j in range(4)) { + var signCol : LLVMOpaqueValue? [2] // this block's sign bytes, rows qd*rv.. at stride 4 - the gemv negates per vector, the tile's panel arrives signed + if (gather && !(pairs && te.vbmi) && (!pairs || te.kq == 33 || te.kq == 23 || te.dotKind != DOT_SDOT)) { + for (qd in range(rq)) { + signCol[qd] = kq_sign_bytes(te, wbP, blk, qd, "sgc{blk}_{qd}") + } + } + if (pairs && te.vbmi) { + emit_grid_rows_blk_vbmi(te, wbP, xb, blk, a, a1, iq2) + } elif (pairs) { + te.gridRows = true + emit_grid_rows_blk(te, wbP, xb, blk, signCol, a, a1, iq2) + te.gridRows = false + } + for (j in range(pairs ? 0 : 4)) { for (qd in range(rq)) { var noff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 16 + j * 4) * mr + qd * w8)), "") var wlo : LLVMOpaqueValue? @@ -1881,6 +2234,10 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var var hoff = LLVMBuildAdd(b, noff, te.types->ConstI64(uint64(128 * mr)), "") wlo = load_vec(te, te.vwi8, wbase, noff, "wlo{blk}_{j * rq + qd}") whi = load_vec(te, te.vwi8, wbase, hoff, "whi{blk}_{j * rq + qd}") + if (signCol[qd] != null) { + wlo = apply_sign_col(te, wlo, signCol[qd], j / 2, j % 2, "wlo{blk}_{j * rq + qd}s") + whi = apply_sign_col(te, whi, signCol[qd], 2 + j / 2, j % 2, "whi{blk}_{j * rq + qd}s") + } } else { var nv = load_vec(te, te.vwi8, te.wg, noff, "nv{blk}_{j * rq + qd}") var loIdx = LLVMBuildAnd(b, nv, splat_i8w(te, 15), "nib.lo") @@ -2194,6 +2551,26 @@ def private setup_tile_emit(var te : TileEmit; var gc : LlvmCodeCtx; p : TilePer var redArgTypes <- [te.vni32] te.reduce_ty = LLVMFunctionType(te.types.t_int32, redArgTypes) } + te.pf = p.pf + if (te.pf != 0) { + var pty = LLVMPointerType(te.types.t_int8, 0u) + var pfTypes <- [pty] + te.prefetch_decl = LLVMGetIntrinsicDeclaration(mod, LLVMLookupIntrinsicID("llvm.prefetch"), pfTypes) + if (te.prefetch_decl == null) return false + var pfArgTypes <- [pty, te.types.t_int32, te.types.t_int32, te.types.t_int32] + te.prefetch_ty = LLVMFunctionType(te.types.t_void, pfArgTypes) + } + te.vbmi = p.grid == "vbmi" + if (te.vbmi) { + var b3Types <- [te.vwi8, te.vwi8, te.vwi8] + te.permi2b_decl = LLVMGetIntrinsicDeclaration(mod, LLVMLookupIntrinsicID("llvm.x86.avx512.vpermi2var.qi.512"), noTypes) + te.mshift_decl = LLVMGetIntrinsicDeclaration(mod, LLVMLookupIntrinsicID("llvm.x86.avx512.pmultishift.qb.512"), noTypes) + te.shufb_decl = LLVMGetIntrinsicDeclaration(mod, LLVMLookupIntrinsicID("llvm.x86.avx512.pshuf.b.512"), noTypes) + if (te.permi2b_decl == null || te.mshift_decl == null || te.shufb_decl == null) return false + te.permi2b_ty = LLVMFunctionType(te.vwi8, b3Types) + te.mshift_ty = LLVMFunctionType(te.vwi8, b2Types) + te.shufb_ty = LLVMFunctionType(te.vwi8, b2Types) + } if (te.dotKind == DOT_MADDUBS) { let maddId = LLVMLookupIntrinsicID(te.width == 256 ? "llvm.x86.avx2.pmadd.ub.sw" : "llvm.x86.avx512.pmaddubs.w.512") te.madd_decl = LLVMGetIntrinsicDeclaration(mod, maddId, noTypes) @@ -3067,19 +3444,9 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // iq3s/iq3xxs/iq2s/iq2xs: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) te.gfn = gc.impl te.gctx = gc.jit.ctx - if (fmt == 33) { - iq3s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } elif (fmt == 23) { - iq2s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } elif (fmt == 24) { - iq2xs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } elif (fmt == 25) { - iq2xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } else { - iq3xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } - te.iq3s_panel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "iq3s.panel") - LLVMSetAlignment(te.iq3s_panel, 32u) + grid_emit_globals(te, fmt, LLVMGetGlobalParent(gc.impl)) + te.gridPanel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "grid.panel") + LLVMSetAlignment(te.gridPanel, 64u) // a cache line: at 32, half the width-512 panel loads straddle one } var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 88d36e48b9..642ae1458d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 26 // the iq2xxs plane pair (IQ2_XXS native tier) +let IMAGE_VERSION = 28 // k6's qh and k3's qs/hmask planes packed per sub-block (CPU flavor) //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 920696275b..2be0a52f39 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -914,6 +914,7 @@ def q8_region_to_metal_blob(qp : int8 const?; sp : float const?; n, rows : int64 // Gather one PREPARED K-quant weight stack into the GPU tier's device planes — quant payloads // row-major per superblock in the k/k+16 nibble pairing, plus DECODED 20B scale rows. Reads // whatever the load prepared (grp interleave or disk-order); device bytes are identical either way. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice_rows : int64; repacked : bool; mr : int64; var wq : array; var ws : array) { let nsb = n / KQ_SUPERBLOCK_ELEMS @@ -968,12 +969,23 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(96l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.k3) { // 16 qs columns, then the 8 hmask columns at 64*mr + } elif (fmt == KqFmt.k3) { // the sub-block-packed qs + hmask columns for (m in range64(64l)) { - dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + let half = (m % 32l) / 16l + let j = (m % 16l) / 4l + var qbyte = 0u + for (bb in range64(4l)) { + let col = ((m / 32l) * 4l + bb) * 2l + half + qbyte |= ((uint(gq[(col * mr + r) * 4l + m % 4l]) >> uint(2l * j)) & 3u) << uint(2l * bb) + } + dq[m] = uint8(qbyte) } for (m in range64(32l)) { - dq[64l + m] = gq[64l * mr + ((m / 4l) * mr + r) * 4l + m % 4l] + var hbyte = 0u + for (blk in range64(8l)) { + hbyte |= ((uint(gq[64l * mr + (blk * mr + r) * 4l + m % 4l]) >> uint(m / 4l)) & 1u) << uint(blk) + } + dq[64l + m] = uint8(hbyte) } } elif (fmt == KqFmt.k2) { // 16 qs columns, nothing else for (m in range64(64l)) { @@ -1000,9 +1012,16 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (bj in range64(32l)) { dq[128l + bj] = gq[128l * mr + bj * mr + r] } - } elif (fmt == KqFmt.k6) { + } elif (fmt == KqFmt.k6) { // the sub-block-packed qh columns for (p in range64(64l)) { - dq[128l + p] = gq[128l * mr + ((p / 4l) * mr + r) * 4l + p % 4l] + let half = (p % 32l) / 16l + let j = (p % 16l) / 4l + var qhbyte = 0u + for (bb in range64(4l)) { + let col = ((p / 32l) * 4l + bb) * 2l + half + qhbyte |= ((uint(gq[128l * mr + (col * mr + r) * 4l + p % 4l]) >> uint(2l * j)) & 3u) << uint(2l * bb) + } + dq[128l + p] = uint8(qhbyte) } } if (fmt == KqFmt.k6 || fmt == KqFmt.k3) { diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 868c319512..1bd8bf100a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -500,7 +500,7 @@ def public get_matmul_min_chunk_rows_gemv() : int { return g_matmul_min_chunk_ro // GEMV tail: the one-chunk-per-lane equal split leaves the join waiting out the SLOWEST lane's // whole chunk (median lane idles 14-24% of every GEMV stage, per-lane trace 3990X). >1 publishes -// wave-aligned chunks per lane so finished lanes keep serving; default 8; 1 = historical equal split. +// wave-aligned chunks per lane so finished lanes keep serving (only bites past 64*waves*lanes rows); 1 = equal split. var g_gemv_chunks_per_lane = 8 def public set_gemv_chunks_per_lane(v : int) { g_gemv_chunks_per_lane = max(v, 1) } def public get_gemv_chunks_per_lane() : int { return g_gemv_chunks_per_lane } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 126d1f932b..a79877b359 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -1057,7 +1057,8 @@ def private dequant_iq4xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } } -//! k3 grp row dequant: the 2-bit lanes and hmask bits stay in place per (h, column) — w = (d * sc16) * (q - 4). +//! w = (d * sc16) * (q - 4) +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def private dequant_k3_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { unsafe { for (sbi in range64(nsb)) { @@ -1066,14 +1067,12 @@ def private dequant_k3_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, ns let d = f16_to_f32(uint(ksg[sb + 16l * mr + 2l * r]) | (uint(ksg[sb + 16l * mr + 2l * r + 1l]) << 8u)) for (blk in range64(8l)) { let kb = sbi * 256l + blk * 32l - let h = blk / 4l - let sh = int((blk % 4l) * 2l) let s0 = d * float(int(int8(ksg[sb + 2l * blk * mr + r]))) let s1 = d * float(int(int8(ksg[sb + (2l * blk + 1l) * mr + r]))) for (l4 in range64(8l)) { for (t in range64(4l)) { - let q2 = (int(kqg[qb + ((h * 8l + l4) * mr + r) * 4l + t]) >> sh) & 3 - let hb = (int(kqg[qb + 64l * mr + (l4 * mr + r) * 4l + t]) >> int(blk)) & 1 + let q2 = (int(kqg[qb + ((blk * 2l + l4 / 4l) * mr + r) * 4l + t]) >> int((l4 % 4l) * 2l)) & 3 + let hb = (int(kqg[qb + 64l * mr + (blk * mr + r) * 4l + t]) >> int(l4)) & 1 dst[kb + l4 * 4l + t] = (l4 < 4l ? s0 : s1) * float((q2 | (hb << 2)) - 4) } } @@ -1293,6 +1292,7 @@ def private dequant_iq2xxs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr, n : int64; var dst : float?) { // nolint:STYLE037,STYLE038 — the flat per-format dispatch (one arm per kq format) is the honest shape let nsb = n / 256l let qsb = kq_qsb(int(fmt)) @@ -1360,7 +1360,7 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m } for (j in range64(4l)) { let hb = fmt == 5l ? int(kqg[qb + 128l * mr + (blk * 4l + j) * mr + r]) : 0 - let g2 = uint((blk % 4l) * 2l) + let sj = uint(2l * j) for (t in range64(4l)) { let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) var qlo = int(nib & 15u) @@ -1369,11 +1369,11 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m qlo |= ((hb >> int(t)) & 1) << 4 qhi |= ((hb >> int(4l + t)) & 1) << 4 } elif (fmt == 6l) { - let hbase = qb + 128l * mr + (blk / 4l) * 32l * mr - let h0 = uint(kqg[hbase + (j * mr + r) * 4l + t]) - let h1 = uint(kqg[hbase + ((j + 4l) * mr + r) * 4l + t]) - qlo |= int(((h0 >> g2) & 3u) << 4u) - qhi |= int(((h1 >> g2) & 3u) << 4u) + let hbase = qb + 128l * mr + blk * 2l * mr * 4l + let h0 = uint(kqg[hbase + r * 4l + t]) + let h1 = uint(kqg[hbase + (mr + r) * 4l + t]) + qlo |= int(((h0 >> sj) & 3u) << 4u) + qhi |= int(((h1 >> sj) & 3u) << 4u) } if (fmt == 6l) { dst[kb + j * 4l + t] = s0 * (float(qlo) - 32.0) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 27f4e2932d..46f5368c03 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -162,6 +162,7 @@ def mx4q8_tile_gen(var yp : float?; wn : uint8 const?; we : uint8 const?; xqp : //! One row's dot off the grp K-quant planes, scalar — the kq stubs' reference body (runs //! only when the stamp declined) and the tests' repack oracle; bit-exact vs the portable disk //! dots. Activations are Q8_K-form: xsp per-superblock scale, xbsp per-16 quant sums. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def kq_grp_row_dot(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { var acc = 0.0 let nsb = n / 256l @@ -179,7 +180,7 @@ def kq_grp_row_dot(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : var ihi = 0 for (j in range64(4l)) { let hb = fmt == 5l ? int(kqg[qb + 128l * mr + (blk * 4l + j) * mr + r]) : 0 - let g2 = uint((blk % 4l) * 2l) + let sj = uint(2l * j) for (t in range64(4l)) { let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) var qlo = int(nib & 15u) @@ -188,11 +189,11 @@ def kq_grp_row_dot(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : qlo |= ((hb >> int(t)) & 1) << 4 qhi |= ((hb >> int(4l + t)) & 1) << 4 } elif (fmt == 6l) { - let hbase = qb + 128l * mr + (blk / 4l) * 32l * mr - let h0 = uint(kqg[hbase + (j * mr + r) * 4l + t]) - let h1 = uint(kqg[hbase + ((j + 4l) * mr + r) * 4l + t]) - qlo |= int(((h0 >> g2) & 3u) << 4u) - qhi |= int(((h1 >> g2) & 3u) << 4u) + let hbase = qb + 128l * mr + blk * 2l * mr * 4l + let h0 = uint(kqg[hbase + r * 4l + t]) + let h1 = uint(kqg[hbase + (mr + r) * 4l + t]) + qlo |= int(((h0 >> sj) & 3u) << 4u) + qhi |= int(((h1 >> sj) & 3u) << 4u) } ilo += qlo * int(xqp[b * 32l + j * 4l + t]) ihi += qhi * int(xqp[b * 32l + 16l + j * 4l + t]) @@ -404,14 +405,12 @@ def k3_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : var bsum = 0 for (blk in range64(8l)) { let b = sbi * 8l + blk - let h = blk / 4l - let sh = int((blk % 4l) * 2l) var ilo = 0 var ihi = 0 for (l4 in range64(8l)) { for (t in range64(4l)) { - let q2 = (int(kqg[qb + ((h * 8l + l4) * mr + r) * 4l + t]) >> sh) & 3 - let hb = (int(kqg[qb + 64l * mr + (l4 * mr + r) * 4l + t]) >> int(blk)) & 1 + let q2 = (int(kqg[qb + ((blk * 2l + l4 / 4l) * mr + r) * 4l + t]) >> int((l4 % 4l) * 2l)) & 3 + let hb = (int(kqg[qb + 64l * mr + (blk * mr + r) * 4l + t]) >> int(l4)) & 1 let q = q2 | (hb << 2) if (l4 < 4l) { ilo += q * int(xqp[b * 32l + l4 * 4l + t]) @@ -534,6 +533,7 @@ def q40q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k4q8_gemv_gen", gen = "dasllama_gemm_gen::k4_gemv"), tune_companion(fn = "k4q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k4_tile", @@ -554,6 +554,7 @@ def k4q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k5q8_gemv_gen", gen = "dasllama_gemm_gen::k5_gemv"), tune_companion(fn = "k5q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k5_tile", @@ -574,6 +575,7 @@ def k5q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k6q8_gemv_gen", gen = "dasllama_gemm_gen::k6_gemv"), tune_companion(fn = "k6q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k6_tile", @@ -594,6 +596,7 @@ def k6q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "q40q8_gemv_gen", gen = "dasllama_gemm_gen::q40_gemv"), tune_companion(fn = "q40q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q40_tile", @@ -626,6 +629,7 @@ def iq4xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq4xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq4xs_gemv"), tune_companion(fn = "iq4xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq4xs_tile", @@ -658,6 +662,7 @@ def k3q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k3q8_gemv_gen", gen = "dasllama_gemm_gen::k3_gemv"), tune_companion(fn = "k3q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k3_tile", @@ -690,6 +695,8 @@ def iq3sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq3sq8_gemv_gen", gen = "dasllama_gemm_gen::iq3s_gemv"), tune_companion(fn = "iq3sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3s_tile", @@ -765,6 +772,8 @@ def iq3xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq3xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq3xxs_gemv"), tune_companion(fn = "iq3xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3xxs_tile", @@ -828,6 +837,7 @@ def iq4nlq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq4nlq8_gemv_gen", gen = "dasllama_gemm_gen::iq4nl_gemv"), tune_companion(fn = "iq4nlq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq4nl_tile", @@ -897,6 +907,7 @@ def k2q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k2q8_gemv_gen", gen = "dasllama_gemm_gen::k2_gemv"), tune_companion(fn = "k2q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k2_tile", @@ -969,6 +980,8 @@ def iq2sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq2sq8_gemv_gen", gen = "dasllama_gemm_gen::iq2s_gemv"), tune_companion(fn = "iq2sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2s_tile", @@ -1040,6 +1053,8 @@ def iq2xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq2xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xs_gemv"), tune_companion(fn = "iq2xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xs_tile", @@ -1113,6 +1128,8 @@ def iq2xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq2xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xxs_gemv"), tune_companion(fn = "iq2xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xxs_tile", @@ -1173,6 +1190,7 @@ def q8q8_tile_s16_gen(var yp : float?; wg : int8 const?; sg : uint16 const?; xqp tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, gkstep = 2, bias = 128, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, prefetch = 2048, requires = "avx512vnni,avx512bw"), // the crown with a 2 KB software prefetch on the gemv stream (lab row) tune_perm(dot = "vpdpbssd", width = 256, mr = 8, kstep = 2, requires = "avxvnniint8"), tune_perm(dot = "smmla", mr = 4, kstep = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 4, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 4, nrsplit = 2, requires = "i8mm"), @@ -1309,6 +1327,7 @@ def q51q8_gemv_gen(var yp : float?; qgp : uint8 const?; sgp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "q51q8_gemv_gen", gen = "dasllama_gemm_gen::q51_gemv"), tune_companion(fn = "q51q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q51_tile", @@ -2074,10 +2093,14 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let packed = kq_reads_packed_planes(int64(fmt)) var scratch : array if (!packed) { - scratch |> resize(mr * nsb * 256l) + scratch |> reserve(mr * nsb * 256l + 64l) + scratch |> resize(mr * nsb * 256l + 64l) // + a cache line: the panel starts 64-aligned, the tile's 32/64-byte loads never straddle one } unsafe { var scp = !packed ? addr(scratch[0]) : reinterpret(null) + if (!packed) { + scp += int64((64ul - (intptr(scp) & 63ul)) & 63ul) + } var tb = tb0 while (tb < tend) { let tbe = min(tb + TB, tend) diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index b44d1470dd..cdaa01fd61 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -56,7 +56,9 @@ def repack_q8q8_grp(var wp : int8?; var sp : float?; n, d, mr : int64; wbias : i let ncg = n / kgroup var tw : array var ts : array + tw |> reserve(d * n) tw |> resize(d * n) + ts |> reserve(d * nb) ts |> resize(d * nb) unsafe { var twp = addr(tw[0]) @@ -102,7 +104,9 @@ def repack_mx4_grp(var np : uint8?; var ep : uint8?; n, d, mr : int64) { let ng = d / mr var tn : array var te : array + tn |> reserve(d * nrow) tn |> resize(d * nrow) + te |> reserve(d * nbb) te |> resize(d * nbb) unsafe { var tnp = addr(tn[0]) @@ -144,7 +148,9 @@ def repack_q51_grp(var qp : uint8?; var sp : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * nb * 20l) tq |> resize(d * nb * 20l) + ts |> reserve(d * nb * 4l) ts |> resize(d * nb * 4l) unsafe { var tqp = addr(tq[0]) @@ -194,7 +200,9 @@ def repack_k4_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -240,7 +248,9 @@ def repack_k5_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -283,6 +293,7 @@ def repack_k5_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def repack_k6_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let nsb = n / 256l let qrow = nsb * 192l @@ -290,7 +301,9 @@ def repack_k6_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -314,10 +327,15 @@ def repack_k6_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { } } } - for (h in range64(2l)) { // qh bytes verbatim, row-interleaved by 4-byte columns - for (jj in range64(8l)) { + for (blk in range64(8l)) { // qh re-packed per sub-block + for (half in range64(2l)) { for (t in range64(4l)) { - kq[dq + 128l * mr + ((h * 8l + jj) * mr + r) * 4l + t] = tqp[sq + 128l + h * 32l + jj * 4l + t] + var qhbyte = 0u + for (j in range64(4l)) { + let l = (blk / 4l) * 32l + half * 16l + j * 4l + t + qhbyte |= ((uint(tqp[sq + 128l + l]) >> uint(2l * (blk % 4l))) & 3u) << uint(2l * j) + } + kq[dq + 128l * mr + ((blk * 2l + half) * mr + r) * 4l + t] = uint8(qhbyte) } } } @@ -344,7 +362,9 @@ def repack_q40_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -384,7 +404,9 @@ def repack_iq4xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -419,9 +441,8 @@ def repack_iq4xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } -//! k3 grp layout: the qs bytes as 16 four-byte columns ([h 0..1][8 l-groups]) x mr rows, then the -//! hmask as 8 columns x mr rows at 64*mr — the 2-bit lanes and mask bits stay in place (the -//! kernels shift per block); scales interleave as k6's row ([16 x mr int8][mr x f16 d]). +//! scales interleave as k6's row ([16 x mr int8][mr x f16 d]) +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let nsb = n / 256l let qrow = nsb * 96l @@ -429,7 +450,9 @@ def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -443,14 +466,23 @@ def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { for (r in range64(mr)) { let sq = (g * mr + r) * qrow + sbi * 96l let ss = (g * mr + r) * srow + sbi * 18l - for (c in range64(16l)) { - for (t in range64(4l)) { - kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + for (blk in range64(8l)) { // qs and hmask re-packed per sub-block + for (half in range64(2l)) { + for (t in range64(4l)) { + var qbyte = 0u + for (j in range64(4l)) { + let l = (blk / 4l) * 32l + half * 16l + j * 4l + t + qbyte |= ((uint(tqp[sq + l]) >> uint(2l * (blk % 4l))) & 3u) << uint(2l * j) + } + kq[dq + ((blk * 2l + half) * mr + r) * 4l + t] = uint8(qbyte) + } } - } - for (c in range64(8l)) { for (t in range64(4l)) { - kq[dq + 64l * mr + (c * mr + r) * 4l + t] = tqp[sq + 64l + c * 4l + t] + var hbyte = 0u + for (sidx in range64(8l)) { + hbyte |= ((uint(tqp[sq + 64l + sidx * 4l + t]) >> uint(blk)) & 1u) << uint(sidx) + } + kq[dq + 64l * mr + (blk * mr + r) * 4l + t] = uint8(hbyte) } } for (idx in range64(16l)) { @@ -476,7 +508,9 @@ def repack_iq3s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -521,7 +555,9 @@ def repack_k2_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -565,7 +601,9 @@ def repack_iq2s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -609,7 +647,9 @@ def repack_iq2xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -653,7 +693,9 @@ def repack_iq2xxs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -746,7 +788,9 @@ def repack_iq3xxs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -950,6 +994,7 @@ def unpack_iq2xxs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int6 } } +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { if (fmt == 33l) { unpack_iq3s_panel_grp(kqg, dst, mr, nsb) @@ -981,9 +1026,9 @@ def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, n let qb = sbi * (fmt == 5l ? 160l : 192l) * mr let ob = sbi * 256l * mr for (blk in range64(8l)) { - let g2 = uint64((blk % 4l) * 2l) - let hbase = qb + 128l * mr + (blk / 4l) * 32l * mr // k6 high plane (k5 ignores) + let hbase = qb + 128l * mr + blk * 2l * mr * 4l // k6 high plane (k5 ignores) for (j in range64(4l)) { + let sj = uint64(2l * j) let gcol = ((blk * 4l + j) * mr) * 4l var r4 = 0l while (r4 < mr) { @@ -1007,12 +1052,12 @@ def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, n hiw[w] = ((n >> 4ul) & LO4) | dh } } else { - let h0w = reinterpret(kqg + (hbase + (j * mr + r4) * 4l)) - let h1w = reinterpret(kqg + (hbase + ((j + 4l) * mr + r4) * 4l)) + let h0w = reinterpret(kqg + (hbase + r4 * 4l)) + let h1w = reinterpret(kqg + (hbase + (mr + r4) * 4l)) for (w in range64(2l)) { let n = nw[w] - low[w] = (n & LO4) | (((h0w[w] >> g2) & TOP2) << 4ul) - hiw[w] = ((n >> 4ul) & LO4) | (((h1w[w] >> g2) & TOP2) << 4ul) + low[w] = (n & LO4) | (((h0w[w] >> sj) & TOP2) << 4ul) + hiw[w] = ((n >> 4ul) & LO4) | (((h1w[w] >> sj) & TOP2) << 4ul) } } r4 += 4l diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 97abd94787..37f23090ac 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -707,14 +707,6 @@ the expected crowned entries' counts moved, once the cell picks a carrier whose shapes actually ready the tensor variants. -61. **An in-process image-off seam for facade-knob test cells.** A cell whose subject is a - facade lane knob (`load_asr_model` under `set_asr_tower_fp32`) must keep the facade - loader, and on a cold box that load BAKES the pinned lane's `.dlim` and GC-purges the - serving lane's flavor (`dlim_gc_stale`) - the class that broke `test_audio_embedder` - once already. The tests checklist ledgers the residue; the fix is an in-process - equivalent of `DASLLAMA_IMAGE=0` (`g_env_engine.image` is a `let` read at load), so such - cells can run image-free instead of risking the purge. - 57. **Plane types have no `long_length`.** `length(PlaneF)` / `length(PlaneU16)` return `int`, so every `uint64(length(t.blob) * 4l)` spelling caps a plane at 2^31 elements before the widening - headroom-only today (whisper large-v3's twin is ~632M elements). @@ -761,16 +753,36 @@ `[tune_perm]` spellings and let the probe judge. Done = a per-format note naming what was raced and what won, beside the existing bench rows. -61. **IQ3_S CPU decode: race a no-panel gemv spelling (the 0.92x tail).** The stamped gemv - gathers each superblock into an alloca panel and then runs the vector dot - a store/load - round trip per superblock that a single token never amortizes; llama.cpp's per-row form - (grid words composed straight into vectors, signs applied to the ACTIVATION via - shuffle+cmpeq/xor-sub, magnitudes kept unsigned for maddubs) edges it 57.0 vs 52.4 tg128 - on the zen2 (pp512 is ours 4.93x - the panel amortizes across the tile). The counter to - race as a [tune_perm]: compose the gathered words directly into the weight vectors - (insertelement per i32 lane, no panel), and/or the signs-on-activation form that drops - the abs+psign pair. Done = a gemv perm that takes tg128 at or past llama.cpp's, crowned - by the probe. +61. **Grid-format CPU decode: the panel round trip, one shared cost (kernel-level 0.43x-0.91x).** + Measured 2026-09-01 at one thread, m=4096 k=14336 (`benchmarks/matmul/kq_kernel_bench.das` vs + the reference exe's `test-backend-ops perf`): iq3s 11406 us vs 10340 (0.91x), iq3xxs 11578 vs + 6590 (0.57x), iq2s 11732 vs 5074 (0.43x), iq2xs 11490 vs 5386 (0.47x), iq2xxs 11061 vs 5124 + (0.46x) - all five at 48-51 ns per superblock against k4/k2's 8.6-8.8, a flat cost independent + of the format. Every one of the five gathers ends each decoded dword with a 4-byte store into the + alloca panel and the dot reloads 32 bytes spanning eight such stores - a wide load over narrow + stores, which x86 does not forward; llama.cpp composes grid words into registers on both arches + (`plans/kernel_parity_research_cpu.md`). Signs-on-activation does NOT port to our 8-rows-per-vector + layout (llama.cpp's own arm64 arms sign the weights for the same reason). The spellings, in + order: `gather="reg"` (group-major compose, insertelement per row, no panel), `psign="mask"` (a + vector sign mask replacing GPR sign math + abs), a `vpdpbssd` seat for AVX-VNNI-INT8 hosts, a + repack-baked parity sign byte. Done = each of the five at or past the reference row at one thread, + crowned by the probe; plan and fact base: `plans/kernel_parity_pass.md`. + 2026-09-01: `gather="reg"` measured 1.85x SLOWER (insertelement chains, 5x the code) and was + dropped; `sign="vec"` landed for iq3s (7732 us = 1.34x the reference) and iq2s (7076 us = + 0.72x). Then all five: the sign column synthesized from the 7-bit codes (parity = the 8th bit, + no plane change), the iq2 formats' u64 grid pair as one load, and the knob collapsed into the + one gemv path (the tuner races the tile, where a gemv-only spelling ties). Kernel-level, one + thread: iq3s 1.37x, iq2s 0.98x, iq2xxs 0.98x, iq2xs 0.86x, iq3xxs 0.83x of the reference. + Open residue: the per-dword qs byte loads of iq2xs/iq3xxs. + 2026-09-01, ARM: the sdot lattice's decode composes row PAIRS straight from the grid words (two + rows x 8 weights per vector, one u64 grid entry per iq2 row, the ksigns formats signed by a +-1 + table row per 7-bit code, iq3s/iq2s by the mask off the plane's sign column) - M1 one thread: + iq2xs 1.24x, iq2xxs 1.20x, iq3xxs 1.36x, iq2s 1.51x, iq3s 1.15x, all five ahead. x86 is untouched + (zen2 v3: iq3s 1.36x, iq2xs 1.13x, iq2xxs 1.04x, iq2s 1.01x, iq3xxs 1.00x) - its residue stays + the qs byte loads; the M1 lesson (3 loads against 4 NEON ops per cycle, count both) is in the plan. + 2026-09-01, x86: the same row-group form measured 1.2-1.5x SLOWER than the panel on zen2 (iq2xs + 4646 -> 6785 us; llama.cpp's AVX2 insert form does 5320 there) - killed for x86, kept for the sdot + lattice; zen4's 0.77-0.88x waits on a port profile (`plans/kernel_parity_pass.md`, queue). 62. **IQ3_S Metal decode: the ~140 GB/s compose ceiling (tg 0.95x).** Eight GEMV forms raced at n=2048 d=8192 - gather placement x3, gather deleted, signs deleted, llama.cpp's exact @@ -782,3 +794,87 @@ or fusing the sign flip into the staged slab per SITE via a second indexed table. Done = a form that clears 180 GB/s in the dispatch-loop probe (QUIRK 22's harness), or a note proving the ceiling is shared by llama.cpp's own kernel when isolated the same way. + +63. **Reasoning-trace length under quantization: a rollback-and-ban sampler feature (Boris, + 2026-09-01).** Quantization noise raises the entropy of a reasoning model's intermediate steps, + and a model trained to self-verify treats its own noisier steps as suspect - the "wait, let me + recheck" branch fires more often and each firing adds a hundred tokens. The remedy in the field: + detect the reconsideration n-grams (", but wait", "Wait,", "Hmm,", "Actually," - a table of a few + dozen phrases) after they are emitted, rewind the KV cache to the position before the phrase and + resample with the phrase's first tokens banned. The trigger is multi-token, so a per-token logit + bias cannot express it; the rewind is what makes it exact. Ours: the decode loop owns the position + counter, so the rewind is "n_past back k, drop k cache rows" plus a phrase table and a ban list in + the sampler; expose it as a server/CLI knob. Done = the knob, a test that a forced ", but wait" + stream rewinds and continues, and a before/after token count on a 27B reasoning prompt. + 2026-09-01 research (`plans/qwen38_thinking_control.md`): the phrase-level rollback is what + Antislop does and costs 69-96% of throughput; the cheap form that the two 2026 papers measure is a + SINGLE-TOKEN logit penalty on the reconsideration tokens ("wait", "but", "alternatively") at + 12-51% shorter traces with equal or better accuracy - build that first, the rollback only if the + single-token penalty measures short. + +64. **Qwen 3.8 thinking control: expose quick / normal / high (Boris, 2026-09-01).** The model's one + first-party knob is `reasoning_effort` with exactly three legal values - `xhigh` (default), + `medium`, `low` - which the chat template turns into one system-prompt sentence (`medium` injects + nothing; anything else, OpenAI's usual `high` included, raises in the template). No budget exists in + the model; `` and `` are single vocab tokens (248068 / 248069), so an early stop is a + one-token force. What our stack lacks (`plans/qwen38_thinking_control.md`): a `dasllama_arch_qwen38.das` + (the GGUF spec exists), the template's `` prefill in `chatml_chat`, and the server drops + `reasoning_effort`. The three levers, in order: (1) map quick/normal/high -> low/medium/xhigh in the + template's effort sentence and accept `reasoning_effort` in the server (translate `high` to `xhigh` + instead of raising); (2) a reasoning token budget in `SamplingParams` that forces `` - the + counter must see the PROMPT-side `` (the template opens the block), so prefill tokens pass + through the sampler's state; (3) the single-token reconsideration penalty of entry 63. Nobody + publishes what the effort levels cost; measure our three rungs on the 27B before naming them. + +65. **An in-process image-off seam for facade-knob test cells.** A cell whose subject is a + facade lane knob (`load_asr_model` under `set_asr_tower_fp32`) must keep the facade + loader, and on a cold box that load BAKES the pinned lane's `.dlim` and GC-purges the + serving lane's flavor (`dlim_gc_stale`) - the class that broke `test_audio_embedder` + once already. The tests checklist ledgers the residue; the fix is an in-process + equivalent of `DASLLAMA_IMAGE=0` (`g_env_engine.image` is a `let` read at load), so such + cells can run image-free instead of risking the purge. + +66. **x86 hybrid core detection for the lane policy.** `get_num_perf_cores` / `is_slow_tier_compute` + are Apple-only (`job_que.cpp`), so `dasllama_jobque_threads_cap` applies its split - every core + for prefill, perf cores for the decode lanes - on darwin alone. A hybrid Intel client (12th-15th + gen) runs its E-cores as decode team lanes and the join waits on their last chunk; prefill is + fine there (homogeneous ISA, extra compute). Detection is small: Linux counts + `/sys/devices/cpu_core/cpus` against `/sys/devices/cpu_atom/cpus` (hybrid Intel, kernel 5.13+), + Windows reads `EfficiencyClass` from `GetLogicalProcessorInformationEx(RelationProcessorCore)`; + the darwin branch then applies unchanged. No EC2 instance has such a part (server Xeons only); + a Hetzner EX44 (i5-13500, 6P+8E) or EX101 (i9-13900) rents by the hour for the check. + +67. **Per-format decode lane cap on SMT x86.** Figures: `benchmarks/matmul/kq_kernel_bench.das` + (`--team --d 32768 --ntok 0 --base-align 4096`, DAS_JOBQUE_THREADS as stated) against `test-backend-ops perf` + at m=32768/16t on the same box. zen4 (8 cores x 2 SMT) at the engine's decode shape (d=32768, DRAM-streamed): the light formats gain at one lane per core - iq4xs 3116 -> 3004 us, + iq4nl 3049 -> 2946, q8s16 6339 -> 6120, k4 3169 -> 2891 - while the lattice grids halve without + the SMT lanes (iq2xs 1412 -> 2714) and k5 does not move. Two siblings split one core's issue + width on a kernel that is not ALU-bound; the reference streams the same bytes at 84-91 GB/s + where ours sit at 79-82 with every lane busy. The darwin policy (`dasllama_jobque_threads_cap`: + every core for prefill, perf cores for decode) is the right shape but one global cap; here it + has to be per family: the tuner races the gemv seat at physical-core lanes beside the all-lane + race it already runs, stores `decode_lanes` per family in the sidecar, and `matmul_chunks_gemv` + applies the format's cap (its callers know the tensor's KqFmt). Ruled 2026-09-01: zen4's iq4xs + 0.92 / iq4nl 0.94 / k5 0.93-0.97 are accepted as they stand; pinned affinity is normally the + faster arrangement and this may be the one case it is not - decide with a second SMT box in hand. + +68. **iq2xs on arm-i8mm reads 0.90 one-thread / 0.95 at the engine shape - the format's true ARM standing, + not a regression.** Figures: `benchmarks/matmul/kq_kernel_bench.das` (1T default shape; `--team --d 32768` + at 10 lanes) against `test-backend-ops perf` on the same M4 Pro. An earlier mid-arc table read 1.46 (1530 us at 10 lanes), but neither today's emitter + nor that commit's own `dasllama/` sources reproduce it on the same box (both measure ~2350; binary, bench + and reference identical) - the old figure was an artifact of that session. The decode form is settled: the + row-group form beats the panel on ARM for every grid format at both shapes (iq2xs panel 3885 vs rows 2365 + at 10 lanes), so the sdot gate is already right. What would actually lift iq2xs on ARM is a cheaper 9-bit + index path in the row form (its u16 word costs two column-byte reads per site where iq2xxs reads one), or + an ARM analog of the x86 symbol lattice over SMMLA - both open kernel work, not a gate flip. + +69. **A class profile can carry winners a class member cannot run.** `tune_cpu_class()` puts every + AVX512VNNI+BW host in `x86-vnni512`, and that class's shipped profile now carries `grid_vbmi` + winners minted on zen4 (which has VBMI). On a member without VBMI (Cascade Lake, Ice Lake-SP + predecessors), adoption marks the profile complete - nothing races - and every vbmi entry + declines at stamp time to its `fallback=` chain, silently costing those boxes the grid formats' + best seats. Two candidate fixes, unbuilt: split a `x86-vbmi512` class above `x86-vnni512` in + `tune_cpu_class` / `tune_class_chain` (a re-mint renames the profile), or make `profile_try_adopt` + race any family whose adopted winner names a perm `tune_requires_ok` rejects locally (general, + no rename, needs the suffix -> requires mapping surfaced to the adopt layer). The woodpecker + round raised the Cascade Lake case. diff --git a/modules/dasLLAMA/followup_vulkan.md b/modules/dasLLAMA/followup_vulkan.md index a185a036fd..53e1d14956 100644 --- a/modules/dasLLAMA/followup_vulkan.md +++ b/modules/dasLLAMA/followup_vulkan.md @@ -537,13 +537,20 @@ module) is independent and can land any time - it is pure structure. it; a cell that embeds through the vulkan override and compares against the CPU pool proves it. -34. **END-OF-ARC: the pp512 tier class (~0.67-0.70x of llama.cpp on 1B shapes).** Every - sb-format cm2 tile lands in the same band (k4 control 0.67x, iq3s 0.70x on the 5060 Ti) - while tg and the CPU tiers hold parity or better - the gap is the shared batch-GEMM - tier, not any one format's decode. Boris 2026-08-30: this one bothers him at 0.7 - - schedule a dedicated pass at the END of the iquant-formats arc (after the last format - lands), not per-format. Start from the followup 29-32 streamed-layer levers and a - kernel-level probe of the cm2 tile vs llama.cpp's mul_mm_cm2 at matched shapes. +34. **The pp512 gap on every sb format (0.67x-0.90x of llama.cpp on 1B shapes) - not one band, and + not the tile.** The board spreads 1.34x (IQ4_XS 0.90, Q2_K 0.87, Q3_K 0.80, IQ4_NL 0.78, IQ2_XS + 0.77, IQ3_S/IQ2_S 0.70, IQ3_XXS 0.69, Q4_K 0.67) while the Q8 rows sit at 1.00x+, so the format + decode is in it. The cm2 tile itself matches llama.cpp's design point for point - geometry, + workgroup, decode-in-load callback, one coopMatMulAdd per k-step, no k-loop barriers, the same + split-k heuristic (`plans/kernel_parity_research_vk.md`). The real differences: llama.cpp hoists + Q4_K/Q5_K scales into shared memory as ready (d,m) pairs where ours re-extracts from a second + SSBO with a variable shift per decoded element, and its scales sit in the quant block where ours + ride a separate plane. Order of work: (1) split the end-to-end budget - `GGML_VK_PERF_LOGGER=1` + per-shape MUL_MAT rows against our `pfq_ts` roles decide whether the GEMM is the 30% at all (the + chain carries 367 barriers per window); (2) the scale hoist; (3) interleave the scale plane into + the quant block; (4) pad N to the tile width. `harness/vk_gemm_probe.das` already carries the + isolation arms (`ref` = llama.cpp's own coopmat2 blob in our harness, `k6x flat` = compose + without scale reads). Boris 2026-08-30: this one bothers him at 0.7. Plan: `plans/kernel_parity_pass.md`. 35. **The grid-format GEMV workgroup re-stage is a fixed per-workgroup cost - amplified on small models.** Every u64-grid gemv (iq2s 8 KB, iq2xs 4 KB) stages the codebook into diff --git a/modules/dasLLAMA/harness/README.md b/modules/dasLLAMA/harness/README.md index d18e7d5006..8617785fe2 100644 --- a/modules/dasLLAMA/harness/README.md +++ b/modules/dasLLAMA/harness/README.md @@ -8,6 +8,7 @@ instruments - each carries its purpose in its header comment. |---|---| | `parity.das` | dasLLAMA side of the token-for-token oracle check: prompt IDs in -> greedy generated IDs out (`GEN_IDS: ...`). Arch-agnostic - routes through `load_gguf` + `generate`. | | `parity.sh` | Runs the reference oracle and `parity.das` on the same prompt and diffs the two ID streams. | +| `kernel_ladder.sh` | The per-box kernel table: every format's stamped kernel (`benchmarks/matmul/kq_kernel_bench.das`, one thread, no model) against the reference exe's `test-backend-ops perf` at the same shape, decode and prefill rows, joined with the ratio. The reference binary carries the `GGML_BENCH_THREADS` define (a three-line edit of its `N_THREADS`; `backend_ops_shapes.patch` is the older, wider form). | | `oracle/simple_ids.cpp` | The reference oracle - links against a reference-engine build and prints greedy token IDs (CPU, no EOG break) rather than decoded text. | | `quant_eval.das` / `quant_eval_q4.das` | Q8 / Q4 reconstruction-error stats vs the fp32 weights. | | `bpe_test.das` | BPE tokenizer corpus gate against the in-repo `ggml-vocab-llama-bpe` fixtures (no model needed). | diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 3eaa4ea0b1..5256eecc72 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -32,6 +32,8 @@ options _dasllama_internal = true // first). require dasllama/dasllama_math +require dasllama/dasllama_par // maybe_parallel_for_indexed - the gemv seat races at the engine's decode shape +require daslib/jobque_boost require dasllama/dasllama_math_default require dasllama/dasllama_math_aarch64_neon require dasllama/dasllama_math_gen @@ -446,6 +448,33 @@ def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) // the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : (fmt == 23l ? "iq2sq8_tile_gen" : (fmt == 24l ? "iq2xsq8_tile_gen" : (fmt == 25l ? "iq2xxsq8_tile_gen" : "k{fmt}q8_tile_gen"))))))) +//! the gemv companion's own manifest entry: the tile's name with the tier swapped +def kq_gemv_entry(fmt : int64) : string => replace(kq_tile_entry(fmt), "_tile_gen", "_gemv_gen") + +//! the gemv's seat among the tile winner's same-layout rows; "" when the tile winner keeps it +[arch(at = "../ARCHITECTURE_MEASUREMENT.md#gemv-seat")] +def private kq_gemv_seat(fmt : int64; winner : int; names : array; vmr : array; tile, gemv : array) : string { + var cand <- [for (i in range(length(names))); i; where vmr[i] == vmr[winner] && tile[i] < 1.0e29lf] + cand |> sort() $(a, b) => tile[a] < tile[b] + if (length(cand) > 2) { + cand |> resize(2) + } + var pick = winner + for (c in cand) { + if (gemv[c] < gemv[pick]) { + pick = c + } + } + let m = gen_margin() + if (pick != winner && gemv[winner] <= gemv[pick] * m) { + tune_detail("{kq_gemv_entry(fmt)}: {names[pick]} does not beat the tile seat {names[winner]} on the gemv by x{m:0.3f} - the gemv follows the tile\n") + pick = winner + } + pick = gen_sticky(kq_gemv_entry(fmt), names, gemv, pick) + delete cand + return pick == winner ? "" : names[pick] +} + // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame def pack_kq_scale_header(var blkb : array; base : int) { @@ -988,6 +1017,41 @@ def run_kq_gemv(gemv; fx : KqFixture; kq : array; ks : array; mr : } } +//! the streamed fixture's master planes repeated `times` over into a repack buffer: a DRAM-bound plane at the +//! same n (byte-identical row groups), x reused, no reference - a fresh build at that size would transcode +//! millions of superblocks and run a scalar reference gemv over them, minutes per family +def tile_kq_planes(fx : KqFixture; times : int64; var buf : KqBuf) { + let qrow = length(fx.kq) + let srow = length(fx.ks) + buf.q |> reserve(qrow * int(times)) + buf.q |> resize(qrow * int(times)) + buf.s |> reserve(srow * int(times)) + buf.s |> resize(srow * int(times)) + unsafe { + for (t in range64(times)) { + memcpy(addr(buf.q[int(t) * qrow]), addr(fx.kq[0]), qrow) + memcpy(addr(buf.s[int(t) * srow]), addr(fx.ks[0]), srow) + } + } +} + +//! the engine's decode dispatch over the fixture: the engine's own splitter and every lane streaming its share +def run_kq_gemv_team(gemv; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { + let d = (fx.d / mr) * mr + let n = fx.n + unsafe { + var yp = addr(y[0]) + let kqp = addr(kq[0]) + let ksp = addr(ks[0]) + let xqp = addr(fx.xq[0]) + let xsp = addr(fx.xs[0]) + let xbsp = addr(fx.xbs[0]) + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { + invoke(gemv, yp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } + } +} + // k6's signed sub-scales cancel within a row, so element-relative tolerance is too strict at // production shapes (intermediate terms are large, the sum is small) — gate against the // fixture's magnitude instead; a real generator bug (wrong stride/lane/scale) is orders louder @@ -1253,7 +1317,8 @@ def kq_test_family(fmt : int64; kfxs : array) : bool { // and the gemv (streamed + hot decode shapes), print the table, and return the tile-best // suffix ("" = a gate failed). The kq gemv is nrsplit-independent — same-mr rows share the // plane and the gemv shape — so the tile bench decides the family entry. -def kq_tune_family(fmt : int64; var rows_out : table) : string { // nolint:STYLE038 — one family race: shared repacked buffers couple gate, bench and pick +[arch(at = "../ARCHITECTURE_MEASUREMENT.md#gemv-seat")] +def kq_tune_family(fmt : int64; var rows_out : table; var gemv_out : string&) : string { // nolint:STYLE037,STYLE038 — one family race: shared repacked buffers couple gate, bench and the two picks // batch shape: fat STREAMING ffn (weights > the L2 budget, multi-token-block walk) — the shape // class that carries ~90% of prefill time. The old 2048x512x64 kv-projection probe was L2-hot // and crowned hot-shape winners that lose double-digit % at model scale (mr4 vs mr8, M1). @@ -1379,6 +1444,65 @@ def kq_tune_family(fmt : int64; var rows_out : table) : stri } var tnames <- [for (t in tvs); t._0] winner = gen_sticky(kq_tile_entry(fmt), tnames, best, winner) + var tbest : array + tbest |> resize(nv) + for (i in range(nv)) { + tbest[i] = gbest[i] + } + with_job_que() { // the lane count and the team dispatch both need the live jobque + if (get_total_hw_jobs() > 1) { + var kefx <- build_kq_fixture(fmt, 14336l, 512l, 1l) + let TILES = 320l + var ktfx = KqFixture(fmt = kefx.fmt, n = kefx.n, d = kefx.d * TILES, ntok = 1l, nsb = kefx.nsb) + ktfx.xq := kefx.xq + ktfx.xs := kefx.xs + ktfx.xbs := kefx.xbs + var tbuf : KqBuf + tile_kq_planes(kefx, TILES, tbuf) + delete kefx + unsafe { + repack_kq_grp_fmt(fmt, addr(tbuf.q[0]), addr(tbuf.s[0]), ktfx.n, ktfx.d, vmr[winner]) + } + var kyt : array + kyt |> resize(ktfx.d) + for (i in range(nv)) { + tbest[i] = 1.0e30lf + } + var tsamples : array> + tsamples |> resize(nv) + for (r in range(7)) { // the seat takes the MEDIAN: a round that finds the plane in L3 must not crown it + for (vi in range(nv)) { + continue if (vmr[vi] != vmr[winner] || best[vi] >= 1.0e29lf) + for (_w in range(r == 0 ? 3 : 0)) { + run_kq_gemv_team(gtab[tvs[vi]._0], ktfx, tbuf.q, tbuf.s, vmr[vi], kyt) + } + let t0 = ref_time_ticks() + run_kq_gemv_team(gtab[tvs[vi]._0], ktfx, tbuf.q, tbuf.s, vmr[vi], kyt) + tsamples[vi] |> push(double(get_time_usec(t0))) + } + } + for (vi in range(nv)) { + if (!empty(tsamples[vi])) { + sort(tsamples[vi]) + tbest[vi] = tsamples[vi][length(tsamples[vi]) / 2] + } + } + delete tsamples + for (vi in range(nv)) { + if (vmr[vi] == vmr[winner] && best[vi] < 1.0e29lf) { + tune_detail("k{fmt} {tvs[vi]._0}: decode at the engine shape {tbest[vi]} us ({get_total_hw_jobs()} lanes, d={ktfx.d} n={ktfx.n})\n") + } + } + delete kyt + delete tbuf + delete ktfx + } + } + gemv_out = kq_gemv_seat(fmt, winner, tnames, vmr, best, tbest) + if (!empty(gemv_out)) { + rows_out["gemv_seat"] = JV((seat = gemv_out, tile_seat = tvs[winner]._0, gemv_us = gbest[winner], team_us = tbest[winner])) + } + delete tbest delete tnames let wname = tvs[winner]._0 delete keep @@ -2088,24 +2212,30 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - // the kq families tune separately — one manifest entry per format, tile-best wins (the - // gemv is nrsplit-independent, so same-mr rows share its plane and kernel shape). No - // child-process confirm gate here: the kq stamp only moves the kq planes' interleave, - // and the e2e exposure (a tile-optimal mr shaving decode kernel rate) is bounded by the - // decode path staying DRAM-bound — validated end-to-end when the entries first landed. + // the kq families tune separately — the tile-best row is the format's entry, and the gemv takes its + // own entry when its seat differs. No child-process confirm gate here: the kq stamp only moves the + // kq planes' interleave, and the e2e exposure (a tile-optimal mr shaving decode kernel rate) is + // bounded by the decode path staying DRAM-bound — validated end-to-end when the entries first landed. for (fmt in kqSel) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() let entry = kq_tile_entry(fmt) tune_progress_kernel_begin(entry, ROUNDS) var krows : table - let w = kq_tune_family(fmt, krows) + var gw = "" + let w = kq_tune_family(fmt, krows, gw) tune_detail("TUNE_GEN_TIME {entry} family {get_time_usec(kq_t0) / 1000} ms\n") tune_progress_kernel_end(entry, w, empty(w) ? "rejected" : "beats") // gate failure (the broken perm is named above): queue the always-correct reference // body so the scope still COMPLETES — a missing key re-tunes every start pending[entry] = empty(w) ? "reference" : w print("{entry} winner: {empty(w) ? "reference (gate-failure fallback)" : w}\n") + // ALWAYS written: tune_sidecar_merge is upsert-only, so a seat the re-race no longer wants + // must be overwritten with the follow-the-tile spelling, not left standing + pending[kq_gemv_entry(fmt)] = empty(gw) ? (empty(w) ? "reference" : w) : gw + if (!empty(gw)) { + print("{kq_gemv_entry(fmt)} winner: {gw} (the tile seat {w} keeps the tile)\n") + } g_rows[entry] = JV(krows) } if (!noise_gate("mid2")) { diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh new file mode 100644 index 0000000000..92893d80b6 --- /dev/null +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# kernel_ladder.sh - the per-box kernel table: every format's stamped kernel (kq_kernel_bench at +# one thread, normal tune mode = the winner this box ships) against the reference exe's +# test-backend-ops perf at the same shape, decode (n=1) and prefill (n=512) rows, joined with +# the ratio (reference us / ours us; >= 1.0 = ours is faster). No model, no jobque: the kernel +# and nothing else. The bench's arena sits page-aligned (--base-align 4096), the phase the engine's +# image planes give the kernels. The reference binary needs the GGML_BENCH_THREADS define (harness README). +# +# Usage: kernel_ladder.sh [fmt,fmt,...] (default: every format) +# KL_DASLANG the daslang binary (default: this tree's bin/daslang, or bin/Release/daslang.exe on +# Windows - never the global DASLANG, which names another tree's binary on dev boxes) +# LCPP_TBO test-backend-ops (default: $HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe) +# NTOK prefill tokens (default 512; 0 = decode rows only) +# KL_MODULE_CACHE a -module-cache path for the bench start (the AST cache: a cold dasLLAMA compile is +# minutes; keep it outside .jitted_scripts, which measurement scripts wipe) +# ROUNDS interleaved rounds per row (default 5) +# TEAM lanes for the decode rows: the bench dispatches the GEMV the engine's way (--team, +# DAS_JOBQUE_THREADS=N) and the reference runs GGML_BENCH_THREADS=N. Default: the box's +# hardware threads (nproc). SOLO=1 = one thread, one raw call - the kernel table the +# tuner's races correspond to. +# BIG decode rows at d=32768 (a weight above any L3 - the many-lane comparison that +# test-backend-ops perf's repeated 4096-row op does not give; the reference needs the +# m=32768 perf case, a two-line edit beside the thread define - HOW_TO_GET_SIDECAR.md). +# Default 1 with TEAM, 0 with SOLO=1. +# Output: a TSV table on stdout - fmt tier perm ours_us ours_med_us ref_us ratio - plus the box line. +set -e -o pipefail # a failing daslang must not hide behind the join +ROOT=$(cd "$(dirname "$0")/../../.." && pwd) # the tree this script lives in, whatever the env says +DASLANG=${KL_DASLANG:-} +if [ -z "$DASLANG" ]; then + if [ -x "$ROOT/bin/Release/daslang.exe" ]; then DASLANG="$ROOT/bin/Release/daslang.exe"; else DASLANG="$ROOT/bin/daslang"; fi +fi +TBO=${LCPP_TBO:-$HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe} +FMTS=${1:-all} +NTOK=${NTOK:-512} +ROUNDS=${ROUNDS:-5} +SOLO=${SOLO:-0} +if [ "$SOLO" -gt 0 ]; then TEAM=0; BIG=${BIG:-0}; else TEAM=${TEAM:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)}; BIG=${BIG:-1}; fi +ROWS=$(( BIG > 0 ? 32768 : 4096 )) +THREADS=$(( TEAM > 0 ? TEAM : 1 )) +TEAM_ARGS=""; [ "$TEAM" -gt 0 ] && TEAM_ARGS="--team" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# KqFmt name -> ggml type name (the reference's type_a) +ggml_type() { + case "$1" in + q8) echo q8_0;; q8s16) echo q8_0;; k4) echo q4_K;; k5) echo q5_K;; k6) echo q6_K;; q40) echo q4_0;; q51) echo q5_1;; + iq4xs) echo iq4_xs;; k3) echo q3_K;; iq3s) echo iq3_s;; iq3xxs) echo iq3_xxs;; iq4nl) echo iq4_nl;; + k2) echo q2_K;; iq2s) echo iq2_s;; iq2xs) echo iq2_xs;; iq2xxs) echo iq2_xxs;; mx4) echo mxfp4;; + *) echo "";; + esac +} + +echo "# kernel ladder $(date +%F) box=$(hostname) daslang=$DASLANG ref=$TBO ntok=$NTOK rounds=$ROUNDS" >&2 + +# ---- ours: one process, every format, the stamped winner per format ---- +[ -x "$TBO" ] || { echo "kernel_ladder: no test-backend-ops at '$TBO' (set LCPP_TBO)" >&2; exit 1; } +grep -q GGML_BENCH_THREADS "$TBO" || { echo "kernel_ladder: '$TBO' lacks the GGML_BENCH_THREADS define - it would run every core and the ratio column would lie (harness README)" >&2; exit 1; } +set +e +MC_ARGS="" +if [ -n "${KL_MODULE_CACHE:-}" ]; then mkdir -p "$(dirname "$KL_MODULE_CACHE")"; MC_ARGS="-module-cache $KL_MODULE_CACHE"; fi +DAS_TUNE_MODE=normal DAS_JOBQUE_THREADS=$THREADS "$DASLANG" -jit $MC_ARGS "$ROOT/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das" \ + -- --fmt "$FMTS" --d "$ROWS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv --base-align 4096 $TEAM_ARGS > "$WORK/ours.raw" 2> "$WORK/ours.err" +rc=$? +set -e +awk -F'\t' 'NF >= 11' "$WORK/ours.raw" > "$WORK/ours.tsv" || true # the TSV rows; engine and tune notes fall away +if [ $rc -ne 0 ] || [ ! -s "$WORK/ours.tsv" ]; then + echo "kernel_ladder: kq_kernel_bench failed (rc=$rc)" >&2 + tail -20 "$WORK/ours.err" >&2 + exit 1 +fi + +# ---- the reference: one perf run per type at one thread; n=1 and n=512 rows ---- +: > "$WORK/ref.tsv" +for f in $(cut -f1 "$WORK/ours.tsv" | sort -u); do + t=$(ggml_type "$f") + [ -n "$t" ] || { echo "kernel_ladder: no ggml type for '$f'" >&2; continue; } + GGML_BENCH_THREADS=$THREADS "$TBO" perf -b CPU -o MUL_MAT -p "type_a=$t,type_b=f32,m=$ROWS,n=(1|$NTOK)," 2>/dev/null \ + | sed 's/\x1b\[[0-9;]*m//g' | awk -v f="$f" ' + /MUL_MAT\(/ { + n = $0; sub(/.*,n=/, "", n); sub(/,.*/, "", n) + us = $0; sub(/.*runs - */, "", us); sub(/ us\/run.*/, "", us) + print f "\t" (n == 1 ? "gemv" : "tile") "\t" us + }' >> "$WORK/ref.tsv" +done + +# ---- join: ours (fmt tier perm n d ntok best med ...) x ref (fmt tier us) ---- +printf 'fmt\ttier\tperm\tours_us\tours_med_us\tref_us\tratio\n' +awk -F'\t' 'NR == FNR { ref[$1 "\t" $2] = $3; next } + { + key = $1 "\t" $2; r = (key in ref) ? ref[key] : "" + ratio = (r != "" && $7 > 0) ? sprintf("%.2f", r / $7) : "-" + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, $7, $8, (r != "" ? r : "-"), ratio + }' "$WORK/ref.tsv" "$WORK/ours.tsv" diff --git a/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json b/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json index c0cf3da8a0..3c3020b4f5 100644 --- a/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json @@ -1,24 +1,24 @@ { "kernels" : { - "add_inplace" : "vec16_u2", - "cvt_f32_to_f16" : "vec16", + "add_inplace" : "vec8_u2", + "cvt_f32_to_f16" : "vec8_u2", "rope_scaled_neox_tab" : "vec8_u2", - "softmax" : "vec8_u2", "q51q8_tile_gen" : "mr8", + "softmax" : "vec8_u2", "mul_inplace" : "vec8_u2", "quantize_q8_0_bs_into_ptr" : "plain", - "dot_q8kv" : "vec4_u4", "iq2xsq8_tile_gen" : "mr8", + "dot_q8kv" : "vec4_u4", "dot_q8q8" : "vec16", - "quantize_q8kv_row" : "plain", "iq4xsq8_tile_gen" : "mr8", "iq4nlq8_tile_gen" : "mr8", + "quantize_q8kv_row" : "plain", "axpy_f16" : "vec8_u2", "q40q8_tile_gen" : "mr8", "axpy_tq4kv" : "vec8_u2", "cvt_tq4kv_to_f32" : "vec8_u2", "axpy" : "vec8_u2", - "dot_q8q8kv" : "plain", + "dot_q8q8kv" : "vec16", "dot_mx4q8" : "u2", "softmax_sink" : "vec8_u2", "iq2xxsq8_tile_gen" : "mr8", @@ -29,15 +29,15 @@ "cvt_q8kv_to_f32" : "vec8_u2", "axpy_q8kv" : "vec8_u2", "dot_q8q8_f16s" : "vec16", - "q8q8_tile_gen" : "mr8_budget", + "q8q8_tile_gen" : "kstep2", "quantize_q8_0_into_ptr" : "plain", "k4q8_tile_gen" : "mr8", - "gemm_f32_uk_4x16" : "u2", - "dot_q51e" : "vec16", "k5q8_tile_gen" : "mr8", "k6q8_tile_gen" : "mr8", "k3q8_tile_gen" : "mr8", "k2q8_tile_gen" : "mr8", + "gemm_f32_uk_4x16" : "u2", + "dot_q51e" : "vec16", "dot_f16" : "vec8_u2", "cvt_f16_to_f32" : "vec8_u2", "dot" : "vec8_u2", @@ -48,22 +48,22 @@ "dot_q4" : "vec4_u4", "quantize_tq4kv_row" : "plain", "copy_floats" : "vec8_u2", - "rmsnorm" : "vec8" + "rmsnorm" : "vec8_u2" }, "provenance" : { "validation" : "ok", - "noise_probes" : "start cv 0.36%; mid1 cv 0.38%; mid2 cv 0.48%; end cv 0.27%", + "noise_probes" : "start cv 1.24%; mid1 cv 1.35%; mid2 cv 1.70%; end cv 1.43%", "platform" : "darwin", - "noise_floor_cv_pct" : "0.48", + "noise_floor_cv_pct" : "1.70", "features" : "neon;dotprod;i8mm;bf16;fullfp16;lse", "class" : "arm-i8mm", - "written" : "2026-09-01T15:43:02.269Z", + "written" : "2026-09-02T00:17:51.676Z", "validation_demoted" : "0", "mode" : "normal", "dasllama_version" : "15", "origin" : "profile", "noise" : "ok", "arch" : "arm64", - "validation_max_drift_pct" : "1.09" + "validation_max_drift_pct" : "0.27" } } \ No newline at end of file diff --git a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json new file mode 100644 index 0000000000..e3bdde0391 --- /dev/null +++ b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json @@ -0,0 +1,76 @@ +{ + "kernels" : { + "iq4xsq8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "q40q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "add_inplace" : "vec16", + "iq2xsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "cvt_f32_to_f16" : "vec16", + "rope_scaled_neox_tab" : "vec16", + "q51q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "softmax" : "vec8_u2", + "mul_inplace" : "vec16", + "quantize_q8_0_bs_into_ptr" : "vec16_u2", + "iq2xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8kv" : "vec8_u2", + "dot_q8q8" : "vec16_u2", + "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "quantize_q8kv_row" : "plain", + "axpy_f16" : "vec16", + "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "axpy_tq4kv" : "vec8_u2", + "cvt_tq4kv_to_f32" : "vec8_u2", + "axpy" : "vec16", + "dot_q8q8kv" : "vec16_u2", + "dot_mx4q8" : "u2", + "softmax_sink" : "vec8_u2", + "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "dot_q8q8_laneq4x4" : "", + "dot_bf16" : "vec8_u2", + "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "add_scale_inplace" : "vec8_u2", + "cvt_q8kv_to_f32" : "vec16_u2", + "axpy_q8kv" : "vec16_u2", + "dot_q8q8_f16s" : "vec16", + "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", + "k6q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "k4q8_tile_gen" : "dot_vpdpbusd_width256_mr16", + "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k3q8_tile_gen" : "dot_vpdpbusd_width256_mr16", + "iq4nlq8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "k2q8_tile_gen" : "dot_vpdpbusd_width256_mr16", + "quantize_q8_0_into_ptr" : "plain", + "gemm_f32_uk_4x16" : "u2", + "dot_q51e" : "vec16", + "dot_f16" : "vec16", + "cvt_f16_to_f32" : "vec16", + "dot" : "vec16_u2", + "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq3xxsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8tq4kv" : "vec16", + "iq3sq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "scale_inplace" : "vec16", + "dot_q4" : "vec8_u2", + "quantize_tq4kv_row" : "plain", + "copy_floats" : "vec8_u2", + "rmsnorm" : "vec16" + }, + "provenance" : { + "validation" : "ok", + "noise_probes" : "start cv 0.08%; mid1 cv 0.04%; mid2 cv 0.12%; end cv 0.05%", + "platform" : "linux", + "noise_floor_cv_pct" : "0.12", + "features" : "avx2;f16c;fma;sse4.2;avxvnni;avx512f;avx512bw;avx512vl;avx512vnni;avx512vbmi;amx-tile;amx-int8", + "class" : "x86-amx", + "written" : "2026-09-01T23:48:21.884Z", + "validation_demoted" : "0", + "mode" : "normal", + "dasllama_version" : "15", + "origin" : "profile", + "noise" : "ok", + "arch" : "x86_64", + "validation_max_drift_pct" : "6.95" + } +} \ No newline at end of file diff --git a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json index ebedf6abd3..3584d8353e 100644 --- a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json @@ -1,69 +1,69 @@ { - "kernels": { - "add_inplace": "vec8_u2", - "add_scale_inplace": "vec8_u2", - "axpy": "vec8_u2", - "axpy_f16": "vec8_u2", - "axpy_q8kv": "plain", - "axpy_tq4kv": "vec8_u2", - "copy_floats": "vec8_u2", - "cvt_f16_to_f32": "vec8_u2", - "cvt_f32_to_f16": "plain", - "cvt_q8kv_to_f32": "vec8_u2", - "cvt_tq4kv_to_f32": "vec8_u2", - "dot": "vec16", - "dot_bf16": "vec8_u2", - "dot_f16": "vec16", - "dot_mx4q8": "u2", - "dot_q4": "vec8_u2", - "dot_q51e": "vec16", - "dot_q8kv": "vec8_u2", - "dot_q8q8": "u2", - "dot_q8q8_f16s": "vec16", - "dot_q8q8_laneq4x4": "", - "dot_q8q8kv": "u2", - "dot_q8tq4kv": "vec16", - "gemm_f32_uk_4x16": "u2", - "iq2sq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq2xsq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq2xxsq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq3sq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq3xxsq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq4nlq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq4xsq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k2q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k3q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k4q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k5q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k6q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "mul_inplace": "vec8_u2", - "q40q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "q51q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "q8q8_tile_gen": "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", - "quantize_q8_0_bs_into_ptr": "u2", - "quantize_q8_0_into_ptr": "plain", - "quantize_q8kv_row": "plain", - "quantize_tq4kv_row": "plain", - "rmsnorm": "vec8", - "rope_scaled_neox_tab": "vec8_u2", - "scale_inplace": "vec8_u2", - "softmax": "vec8_u2", - "softmax_sink": "vec8_u2" - }, - "provenance": { - "arch": "x86_64", - "class": "x86-vnni512", - "dasllama_version": "15", - "features": "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni", - "mode": "normal", - "noise": "ok", - "noise_floor_cv_pct": "0.21", - "noise_probes": "start cv 0.09%; mid1 cv 0.04%; mid2 cv 0.21%; end cv 0.06%", - "origin": "profile", - "platform": "linux", - "validation": "ok", - "validation_demoted": "3", - "validation_max_drift_pct": "21.77", - "written": "2026-09-01T01:30:04.021Z" - } -} + "kernels" : { + "add_inplace" : "vec8_u2", + "cvt_f32_to_f16" : "plain", + "rope_scaled_neox_tab" : "plain", + "q51q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "softmax" : "vec8_u2", + "mul_inplace" : "vec8_u2", + "quantize_q8_0_bs_into_ptr" : "u2", + "iq2xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "dot_q8kv" : "vec8_u2", + "dot_q8q8" : "u2", + "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "quantize_q8kv_row" : "plain", + "axpy_f16" : "plain", + "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "axpy_tq4kv" : "vec8_u2", + "cvt_tq4kv_to_f32" : "vec8_u2", + "axpy" : "vec8_u2", + "dot_q8q8kv" : "u2", + "dot_mx4q8" : "u2", + "softmax_sink" : "vec8_u2", + "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "dot_q8q8_laneq4x4" : "", + "dot_bf16" : "vec8_u2", + "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "add_scale_inplace" : "vec8_u2", + "cvt_q8kv_to_f32" : "vec8_u2", + "axpy_q8kv" : "plain", + "dot_q8q8_f16s" : "vec16", + "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", + "quantize_q8_0_into_ptr" : "plain", + "k4q8_tile_gen" : "dot_vpdpbusd_width256_mr16", + "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k3q8_tile_gen" : "dot_maddubs_width256_mr8", + "k2q8_tile_gen" : "dot_vpdpbusd_width256_mr16", + "gemm_f32_uk_4x16" : "u2", + "dot_q51e" : "vec16", + "dot_f16" : "vec16", + "cvt_f16_to_f32" : "vec8_u2", + "dot" : "vec16", + "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "dot_q8tq4kv" : "vec16", + "scale_inplace" : "vec8_u2", + "dot_q4" : "vec8_u2", + "quantize_tq4kv_row" : "plain", + "copy_floats" : "vec8_u2", + "rmsnorm" : "vec8" + }, + "provenance" : { + "validation" : "ok", + "noise_probes" : "start cv 0.07%; mid1 cv 0.24%; mid2 cv 0.05%; end cv 0.08%", + "platform" : "linux", + "noise_floor_cv_pct" : "0.24", + "features" : "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni;avx512vbmi", + "class" : "x86-vnni512", + "written" : "2026-09-01T23:49:31.046Z", + "validation_demoted" : "1", + "mode" : "normal", + "dasllama_version" : "15", + "origin" : "profile", + "noise" : "ok", + "arch" : "x86_64", + "validation_max_drift_pct" : "23.00" + } +} \ No newline at end of file diff --git a/modules/dasLLAMA/performance/profile_common.das b/modules/dasLLAMA/performance/profile_common.das index 94e6dc56ef..375342d197 100644 --- a/modules/dasLLAMA/performance/profile_common.das +++ b/modules/dasLLAMA/performance/profile_common.das @@ -315,6 +315,7 @@ def run_and_stream(args : array; var tail_out : string&; keep_tail : int var acc = "" let rc = unsafe(popen_argv(args, 0.0, $(f) { if (f == null) { + to_log(LOG_ERROR, "dasLLAMA bench: could not spawn {args[0]} (no child, no output)\n") return } while (true) { @@ -410,13 +411,7 @@ def run_capture(cmd : string; first_line : bool) : string { // The running daslang's version, via the box's binary (DASLANG_BIN, default bin/daslang from the // repo-root cwd these tools run in). Takes the last token of the first --version line ("0.6.3"). def das_version() : string { - // shell-free spawn (fio's run_and_capture → popen_argv), so no cmd and no quote/option traps; - // CreateProcess still wants backslashes in the program path (probe-verified: `bin/daslang` - // fails to spawn, `bin\daslang` runs), hence the normalize - var bin = g_env_prof.daslang_bin - if (is_windows()) { - bin = replace(bin, "/", "\\") - } + let bin = g_env_prof.daslang_bin var out = "" if (run_and_capture([bin, "--version"], out) != 0) { return "unknown" diff --git a/modules/dasLLAMA/tests/test_dispatch_shaping.das b/modules/dasLLAMA/tests/test_dispatch_shaping.das index 8f1f14af12..536f105a4e 100644 --- a/modules/dasLLAMA/tests/test_dispatch_shaping.das +++ b/modules/dasLLAMA/tests/test_dispatch_shaping.das @@ -24,6 +24,11 @@ def private pin_shaping_baseline() { set_batch_grid_2d(0) } +[test] +def test_gemv_chunks_default(t : T?) { + t |> equal(get_gemv_chunks_per_lane(), 8, "the shipped default - retune the ladder tables if this moves") +} + [test] def test_lanes_for_work(t : T?) { with_job_que() { diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index c1bf709a7a..81da039936 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -6,6 +6,8 @@ options _function_length = 300 // and each gate carries its whole fixture options _dasllama_internal = true require dastest/testing_boost public +require dasllama/dasllama_gemm_gen // vbmi_pack_word / vbmi_alphabet / pm1_of - the lattice's pure table builders +require dasllama/dasllama_layout // moe_gpu_gather_stack_kq - the device-plane gather oracle require _model_tier require dasllama/dasllama_gguf require dasllama/dasllama_common // dequant_kq_plane_sb - the carrier-form dispatcher @@ -2564,3 +2566,142 @@ def test_kq_plane_sb_metal_blob_form(t : T?) { } } } + +// The VBMI lattice's tables are pure host-side data - prove them on any box, no VBMI needed: +// every grid entry's packed code decodes back to the grid's own bytes through the alphabet, +// and the ksigns +-1 expansion matches a scalar re-derivation. +def private grid_words(kq : int) : array { + if (kq == 34) { + return <- [for (w in iq3xxs_grid()); w] + } elif (kq == 33) { + return <- [for (w in iq3s_grid()); w] + } elif (kq == 25) { + return <- [for (w in iq2xxs_grid2()); w] + } elif (kq == 24) { + return <- [for (w in iq2xs_grid2()); w] + } + return <- [for (w in iq2s_grid2()); w] +} + +def private vbmi_plane_gate(t : T?; kq : int) { + let alpha <- vbmi_alphabet(kq) + let iq3 = kq == 33 || kq == 34 + let bits = iq3 ? 3 : 2 + let nbytes = iq3 ? 2 : 4 + let mask = iq3 ? 7 : 3 + let words <- grid_words(kq) + var bad = 0 + for (word in words) { + for (half in range(iq3 ? 2 : 1)) { // iq3: one word = two plane bytes (low/high pair); iq2: one word = one plane byte + let w = half == 0 ? word : (word >> 16u) + let code = vbmi_pack_word(w, alpha, bits, nbytes) + for (tb in range(nbytes)) { + let sym = (code >> (bits * tb)) & mask + let back = alpha[sym] + let orig = int((w >> uint(8 * tb)) & 0xFFu) + if (back != orig) { + bad++ + } + } + } + } + t |> equal(bad, 0, "kq {kq}: every grid byte round-trips through the code plane and alphabet") +} + +[test] +def test_vbmi_lattice_tables(t : T?) { + for (kq in [25, 24, 23, 34, 33]) { + t |> run("vbmi code planes round-trip the kq {kq} grid") @(t : T?) { + vbmi_plane_gate(t, kq) + } + } + t |> run("pm1_of matches the scalar +-1 expansion of every ksigns byte") @(tt : T?) { + var bad = 0 + for (sb in range(256)) { + let packed = pm1_of(sb) + for (k in range(8)) { + let lane = int8(int(packed >> uint64(8 * k)) & 0xFF) + let want = int8((sb & (1 << k)) != 0 ? -1 : 1) + if (lane != want) { + bad++ + } + } + } + tt |> equal(bad, 0) + } +} + +// The GPU device-plane gather's own invariant ("device bytes are identical either way"): gathering +// from disk-order planes and from the grp-repacked planes must produce byte-identical wq/ws. +// This is the only k3/k6 sub-block-plane reader with no dot-product oracle - the byte equality IS its gate. +def private kq_gpu_gather_gate(t : T?; fmt : int) { + let n = 512l + let d = 32l + let nsb = n / 256l + let mr = kq_layout_of(fmt) + let qsb = kq_qsb(fmt) + let ssb = kq_ssb(fmt) + let blk1 <- fmt == 3 ? build_q3k_block() : build_q6k_block() + var kq : array + var ks : array + kq |> resize(d * nsb * qsb) + ks |> resize(d * nsb * ssb) + for (r in range64(d)) { + for (sbi in range64(nsb)) { + if (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + sbi) * qsb, ks, (r * nsb + sbi) * ssb) + } else { + transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + sbi) * qsb, ks, (r * nsb + sbi) * ssb) + } + } + var st = 0xC0FFEEu + uint(r) * 0x9E3779Bu + for (i in range64(nsb * qsb)) { + st = st * 1664525u + 1013904223u + kq[r * nsb * qsb + i] = uint8(uint(kq[r * nsb * qsb + i]) ^ (st >> 16u)) + } + } + var m = Model() + if (fmt == 3) { + m.k3q := kq + m.k3s := ks + } else { + m.k6q := kq + m.k6s := ks + } + var wqDisk : array + var wsDisk : array + moe_gpu_gather_stack_kq(m, fmt == 3 ? KqFmt.k3 : KqFmt.k6, 0l, n, d, d, false, mr, wqDisk, wsDisk) + unsafe { + if (fmt == 3) { + repack_k3_grp(addr(m.k3q[0]), addr(m.k3s[0]), n, d, mr) + } else { + repack_k6_grp(addr(m.k6q[0]), addr(m.k6s[0]), n, d, mr) + } + } + var wqGrp : array + var wsGrp : array + moe_gpu_gather_stack_kq(m, fmt == 3 ? KqFmt.k3 : KqFmt.k6, 0l, n, d, d, true, mr, wqGrp, wsGrp) + var qdiff = 0 + for (a, b in wqDisk, wqGrp) { + if (a != b) { + qdiff++ + } + } + var sdiff = 0 + for (a, b in wsDisk, wsGrp) { + if (a != b) { + sdiff++ + } + } + t |> equal(qdiff, 0, "kq {fmt}: quant device bytes identical from disk-order and grp planes") + t |> equal(sdiff, 0, "kq {fmt}: scale device bytes identical from disk-order and grp planes") +} + +[test] +def test_kq_gpu_gather_grp(t : T?) { + for (fmt in [3, 6]) { + t |> run("moe_gpu_gather_stack_kq kq {fmt}: disk-order and grp planes gather identically") @(t : T?) { + kq_gpu_gather_gate(t, fmt) + } + } +} diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md index 59a878dba9..cae1d0740b 100644 --- a/modules/dasLLVM/ARCHITECTURE.md +++ b/modules/dasLLVM/ARCHITECTURE.md @@ -161,3 +161,34 @@ because the mints that matter most produce no successful read - the first mint h and a re-mint replaces one the gate rejected. An absent file registers as size -1 and hash 0, which the next run's re-validation sees change. Registering is a no-op outside compilation, so the manifest's runtime readers reach the same call unconditionally. + +## 6. The x64 kernel-matrix tier gates {#x64-tier-gates} + +An x64 host target publishes eight boolean gates (`g_target_x64_*`), one per instruction tier the +kernel matrix and the tune grids select on: `avx2`, `f16c`, `vnni256` (256-bit VPDPBUSD by either +VEX AVX-VNNI or EVEX AVX512-VNNI+VL), `avx512bw` (zmm byte ops - BW, not merely F), `avx512vnni` +(zmm VPDPBUSD; implies bw, the sign trick around it is BW), `avx512vbmi` (VPERMI2B / VPERMB / +VPMULTISHIFTQB - the grid formats' symbol lattice), `vnniint8` (VEX VPDPBSSD, native s8 x s8), +and `amx` (both amx-tile and amx-int8; the per-process XTILEDATA grant is a separate runtime step +the family's own witness performs). `init_jit_target_flags` decides each from cpuid truth OR'd with +the `DAS_JIT_X64_FORCE_FEATURES` emission-only override; a cross triple or a generic target +(`host_features = false`) drops to forced-only truth - cpuid is not consulted, the force env is +the only tier source there. + +The cpuid truth is `das_cpu_supports` (`src/builtin/module_builtin_runtime.cpp`), a hand-kept table +keyed by the LLVM target-feature spelling - so the force env and `llc -mattr` take the same names. +A tier feature usually lands as three parts: its cpuid line there, its name in +`TUNE_KNOWN_FEATURES` (`daslib/llvm_tune.das`, the profile fingerprint the `requires=` gates are +checked against), and - when the emitters branch on it - a `g_target_x64_*` gate. The cpuid line +is the load-bearing one: a name missing from the table answers false on every box, so the perm +that requires it declines everywhere and no error names the cause. + +## 7. A constant-folded GEP is not an instruction {#gep-constant-fold} + +`LLVMBuildGEP2` over a global with a constant index does not create an instruction - LLVM folds +it into a `ConstantExpr`, one shared object per distinct expression in the context. Any API that +casts a "just built" GEP to `GetElementPtrInst` therefore writes through the wrong type into the +constant's memory when the fold happened - `LLVMSetIsInBounds` was the instance that corrupted +the context (heap damage surfacing in `LLVMContextDispose` at teardown). The in-bounds form is +requested at build time (`LLVMBuildInBoundsGEP2`), which folds to an in-bounds `ConstantExpr` +correctly; the `llvm_boost` wrapper's `inbounds` default rides that builder. diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 1607024a98..0c4deff6cf 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -71,3 +71,14 @@ recorded path is read by other people and must not name the user who minted it. A host path a diff passes to a filesystem call stays raw: no filesystem call resolves `~`. `tests/llvm_tune_manifest.das` here asserts a minted sidecar carries no home directory. + +- **Never call `LLVMSetIsInBounds` - build the GEP in-bounds with `LLVMBuildInBoundsGEP2` (the + `llvm_boost` wrapper's `inbounds` default) instead.** A constant-folded GEP is a + `ConstantExpr`, and the setter's cast writes through the wrong type into it + (`ARCHITECTURE.md#gep-constant-fold`). + +- **A feature name used in a `requires=` list or a `g_target_x64_*` gate has its cpuid line in + `das_cpu_supports` (`src/builtin/module_builtin_runtime.cpp`) in the same diff** + (`ARCHITECTURE.md#x64-tier-gates`). A name the cpuid table does not know answers false on + every box, so every perm that requires it silently declines to its fallback and no error names + the cause. diff --git a/modules/dasLLVM/daslib/llvm_boost.das b/modules/dasLLVM/daslib/llvm_boost.das index 653232fc0b..a9f98d4d36 100644 --- a/modules/dasLLVM/daslib/llvm_boost.das +++ b/modules/dasLLVM/daslib/llvm_boost.das @@ -307,12 +307,12 @@ def LLVMAddIncoming(phi : LLVMOpaqueValue?; var phi_vals : array starts_with("wasm") @@ -626,6 +627,7 @@ def public init_jit_target_flags(target_triple : string = ""; host_features : bo || (x64_tier_feature("avx512vnni", forced_only) && x64_tier_feature("avx512vl", forced_only))) g_target_x64_avx512bw = g_target_x64_avx2 && x64_tier_feature("avx512f", forced_only) && x64_tier_feature("avx512bw", forced_only) g_target_x64_avx512vnni = g_target_x64_avx512bw && x64_tier_feature("avx512vnni", forced_only) + g_target_x64_avx512vbmi = g_target_x64_avx512bw && x64_tier_feature("avx512vbmi", forced_only) g_target_x64_vnniint8 = g_target_x64_avx2 && x64_tier_feature("avxvnniint8", forced_only) g_target_x64_amx = g_target_is_x64 && x64_tier_feature("amx-tile", forced_only) && x64_tier_feature("amx-int8", forced_only) // target OS truth (native = host platform; cross = the triple's OS field) diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index ca7780097e..0048c0d457 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,11 +36,11 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x59ul // aarch64 hosts append +i8mm when cpu_supports says so (0x58: [hint(unsafe_division_check)] drops the sdiv/srem guards) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x68ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) -let LLVM_JIT_EMITTER_HASH : uint64 = 0xfcc23a40234a227ul +let LLVM_JIT_EMITTER_HASH : uint64 = 0x6a0cebf74a395071ul let JIT_FNV_PRIME : uint64 = 1099511628211ul @@ -176,7 +176,7 @@ def private has_generated_kernel(prog : Program?; jit_all_functions : bool) : bo // The environment/config salt shared by the whole-DLL key AND the per-partition obj cache key. // ONE helper on purpose: a salt component folded into the DLL key but not the partition key // would let a config change reuse stale cached partition objects. -def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; // nolint:STYLE037 - one flat fold per emission input; a split would hide which inputs key the cache +def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; // nolint:STYLE037,STYLE038 - one flat fold per emission input; a split would hide which inputs key the cache target_triple : string; split : bool; use_host_cpu : bool) : uint64 { var h = LLVM_JIT_CODEGEN_VERSION // host/binary identity: object format follows the triple (COFF vs ELF on one checkout), and the emitting binary bakes its Context offsets into every global access - neither is in any AST hash @@ -230,6 +230,12 @@ def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_ if (cpu_supports("avx512vnni")) { isa |= 16ul } + if (cpu_supports("avx512vbmi")) { // the grid formats' symbol lattice emits on it (grid = "vbmi") + h = (h ^ 0x1Bul) * JIT_FNV_PRIME + } + if (cpu_supports("amx-tile") && cpu_supports("amx-int8")) { // tune_cpu_class picks x86-amx, which selects grid decode forms + h = (h ^ 0x1Cul) * JIT_FNV_PRIME + } if (cpu_supports("avxvnniint8")) { isa |= 32ul } diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index 476505a7c7..5a15f921ae 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -1237,11 +1237,12 @@ def tune_pick_fallback(chain : string) : string { return "" } -let private TUNE_KNOWN_FEATURES = "avx2;f16c;fma;sse4.2;avxvnni;avxvnniint8;avx512f;avx512bw;avx512vl;avx512vnni;amx-tile;amx-int8;neon;dotprod;i8mm;bf16;fullfp16;sve;lse" +let private TUNE_KNOWN_FEATURES = "avx2;f16c;fma;sse4.2;avxvnni;avxvnniint8;avx512f;avx512bw;avx512vl;avx512vnni;avx512vbmi;amx-tile;amx-int8;neon;dotprod;i8mm;bf16;fullfp16;sve;lse" //! The ';'-joined subset of the known `requires=` feature names this box passes — stamped //! into every sidecar's provenance as `features`, which is what lets a shipped profile say //! exactly which gated seats its minting box could and could not race. +[arch(at = "../ARCHITECTURE.md#x64-tier-gates")] def tune_feature_fingerprint() : string { var have <- [for (f in split(TUNE_KNOWN_FEATURES, ";")); f; where fallback_feature_ok(f)] let joined = join(have, ";") @@ -1256,7 +1257,7 @@ def tune_cpu_class() : string { let arch = get_architecture_name() if (arch == "x86_64") { if (fallback_feature_ok("avx512vnni") && fallback_feature_ok("avx512bw")) { - return "x86-vnni512" + return (fallback_feature_ok("amx-int8") && fallback_feature_ok("amx-tile")) ? "x86-amx" : "x86-vnni512" } if (fallback_feature_ok("avxvnni")) { return "x86-vnni256" @@ -1273,7 +1274,7 @@ def tune_cpu_class() : string { //! profile adopts the nearest lower class and races only what that class could not. def tune_class_chain() : array { let ladder = (get_architecture_name() == "x86_64" - ? "x86-vnni512;x86-vnni256;x86-avx2;x86-base" + ? "x86-amx;x86-vnni512;x86-vnni256;x86-avx2;x86-base" : "arm-i8mm;arm-neon") let own = tune_cpu_class() var chain : array @@ -1653,6 +1654,18 @@ class private TuneAnnotation : AstFunctionAnnotation { suffix = "" } } + var companionOwnSuffix : array // a companion's OWN entry (the harness's gemv seat) when it names an eligible perm of this grid; else it follows the main + companionOwnSuffix |> resize(length(comps)) + if (manifestFound) { + for (ci, c in count(), comps) { + let own = manifest?["{c.fnname}"] ?? "" + for (p in perms) { + if (p.suffix == own && perm_requires_ok(p)) { + companionOwnSuffix[ci] = own + } + } + } + } let fromManifest = !empty(suffix) var ok = true if (empty(suffix)) { @@ -1685,8 +1698,22 @@ class private TuneAnnotation : AstFunctionAnnotation { for (p in perms) { if (p.suffix == suffix) { stamp_llvm_code(func, genKey, p, fromManifest ? mpath : "") - for (c in comps) { - stamp_llvm_code(c.fn, c.genkey, p, fromManifest ? mpath : "") + for (ci, c in count(), comps) { + var stampedOwn = false + if (!empty(companionOwnSuffix[ci]) && companionOwnSuffix[ci] != suffix) { + for (q in perms) { + if (q.suffix == companionOwnSuffix[ci]) { + stamp_llvm_code(c.fn, c.genkey, q, mpath) + stampedOwn = true + } + } + if (stampedOwn && tune_verbosity() == "verbose") { + print("llvm_tune: {c.fnname} <- {companionOwnSuffix[ci]} (its own entry, {mpath})\n") + } + } + if (!stampedOwn) { + stamp_llvm_code(c.fn, c.genkey, p, fromManifest ? mpath : "") + } } if (tune_verbosity() == "verbose") { print("llvm_tune: {fname} <- {suffix} ({fromManifest ? mpath : "fallback"})\n") @@ -1709,6 +1736,7 @@ class private TuneAnnotation : AstFunctionAnnotation { } } } + delete companionOwnSuffix delete perms delete comps return ok diff --git a/modules/dasLLVM/tests/llvm_gep_inbounds.das b/modules/dasLLVM/tests/llvm_gep_inbounds.das new file mode 100644 index 0000000000..349fa845fb --- /dev/null +++ b/modules/dasLLVM/tests/llvm_gep_inbounds.das @@ -0,0 +1,51 @@ +options gen2 + +require dastest/testing_boost public +require daslib/fio +require strings +require daslib/strings_boost +require daslib/module_path + +// Machine-enforces REVIEW.md's GEP rule: no dasLLVM source outside bindings/ may call +// LLVMSetIsInBounds - a constant-folded GEP is a ConstantExpr, and the setter's cast writes +// the flag through the wrong type into it (ARCHITECTURE.md#gep-constant-fold). Build the GEP +// in-bounds with LLVMBuildInBoundsGEP2 instead. Weakening this test is a defect. + +def private scan_dir(dir : string; var offenders : array) { + fio::dir(dir) $(name : string) { + if (name == "." || name == "..") { + return + } + let path = "{dir}/{name}" + let st = stat(path) + if (st.is_dir) { + if (name != "bindings") { + scan_dir(path, offenders) + } + return + } + if (!(name |> ends_with(".das")) || name == "llvm_gep_inbounds.das") { // the scanner names its own quarry + return + } + let text = fread(path) + var ln = 1 + for (line in split(text, "\n")) { + let cmt = line |> find("//") + let code = cmt >= 0 ? slice(line, 0, cmt) : line // a mention in a comment is not a call + if (code |> find("LLVMSetIsInBounds(") >= 0) { + offenders |> push("{path}:{ln}") + } + ln++ + } + } +} + +[test] +def test_no_llvm_set_is_in_bounds(t : T?) { + var offenders : array + scan_dir("{get_this_module_dir()}/..", offenders) + for (o in offenders) { + t |> failure("LLVMSetIsInBounds called at {o} - build the GEP in-bounds with LLVMBuildInBoundsGEP2 (REVIEW.md; ARCHITECTURE.md#gep-constant-fold)") + } + t |> success(empty(offenders), "no LLVMSetIsInBounds call sites outside bindings/") +} diff --git a/modules/dasLLVM/tests/llvm_tune_manifest.das b/modules/dasLLVM/tests/llvm_tune_manifest.das index 53f314459b..eb743dfcd7 100644 --- a/modules/dasLLVM/tests/llvm_tune_manifest.das +++ b/modules/dasLLVM/tests/llvm_tune_manifest.das @@ -95,6 +95,22 @@ def test_llvm_tune_manifest_roundtrip(t : T?) { // nolint:STYLE038 - flat sequ t |> success(other6) // no sidecar entry -> the fallback stamp (k1) stands t |> success(ref5) // explicit "reference" entry -> the original body despite fallback + // the companion's OWN sidecar entry (the gemv-seat rail): man_kv stamps from it while man_add keeps k2 + var olines : array + let rco = spawn_client("\"{bin}\" -jit \"{clientPath}\" -- write-own", olines) + t |> equal(rco, 0) + var ownLines : array + let rcor = spawn_client("\"{bin}\" -jit \"{clientPath}\"", ownLines) + t |> equal(rcor, 0) + var ownResult7 = false + var ownKv1 = false + for (ln in ownLines) { + ownResult7 ||= ln == "RESULT 7" + ownKv1 ||= ln == "KV 1" + } + t |> success(ownResult7, "the main entry still stamps k2") + t |> success(ownKv1, "the companion stamps from its own entry") + // STALE sidecar (older than the daslang binary) reads as absent: back to fallback tier let backRes = set_mtime_result(sidecarPath, mktime(2000, 1, 1, 0, 0, 0)) t |> success(!(backRes is error), "could not back-date the sidecar") diff --git a/modules/dasLLVM/tests/llvm_tune_manifest_client.das b/modules/dasLLVM/tests/llvm_tune_manifest_client.das index c27d7e70ba..750320fab5 100644 --- a/modules/dasLLVM/tests/llvm_tune_manifest_client.das +++ b/modules/dasLLVM/tests/llvm_tune_manifest_client.das @@ -40,10 +40,16 @@ def main { var n = 2 // nolint:LINT003 — var defeats const-fold so the calls survive to codegen let args <- get_command_line_arguments() // locked view — do not delete var doWrite = false + var doWriteOwn = false for (a in args) { doWrite ||= a == "write" + doWriteOwn ||= a == "write-own" } - if (doWrite) { + if (doWriteOwn) { + // the companion's OWN entry (the gemv-seat shape): man_kv stamps from it, man_add keeps its k2 + let ok = tune_manifest_set("man_kv", "k1") + print("WRITE {ok ? "ok" : "FAILED"} -> {tune_manifest_path()}\n") + } elif (doWrite) { let ok = tune_manifest_set("man_add", "k2") && tune_manifest_set("man_ref", "reference") print("WRITE {ok ? "ok" : "FAILED"} -> {tune_manifest_path()}\n") } else { diff --git a/modules/dasLLVM/tests/llvm_tune_profiles.das b/modules/dasLLVM/tests/llvm_tune_profiles.das index c528c2d0f2..bf80fd6253 100644 --- a/modules/dasLLVM/tests/llvm_tune_profiles.das +++ b/modules/dasLLVM/tests/llvm_tune_profiles.das @@ -137,6 +137,7 @@ def test_llvm_tune_profiles(t : T?) { // nolint:STYLE038 - one flat spawn-and-as // 6. the class chain: only a LOWER class's profile on disk, and the box adopts it (the // residue race then covers whatever its own class unlocks) var chain <- tune_class_chain() + t |> equal(chain[0], tune_cpu_class(), "the box's own class heads its ladder - a class outside the chain could never adopt its own profile") if (length(chain) > 1) { let lower = chain[length(chain) - 1] remove(sidecar) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md new file mode 100644 index 0000000000..2b5b591ed2 --- /dev/null +++ b/plans/kernel_parity_pass.md @@ -0,0 +1,1046 @@ +# Kernel parity pass - CPU and Vulkan kernels to same-speed-or-better + +The arc after PR #3915 (the iquant format ladder). The bar is Boris's: **parity is >= 1.0x of +llama.cpp on the row; 0.95x is a gap.** The method is the compile-profile loop below - never a +model load, a mint, or a 48 s JIT re-emit per try. + +Branch `bbatkin/kernel-parity`, stacked on the PR-1 tip; rebases onto master after that merge. + +## 1. The procedure - three rigs, inner to outer + +1. **Inner loop: the kernel microbench, seconds per try, no model.** + `modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das` - raw kernels, one thread, synthetic + planes, every row of a kernel's `_variants()` registry timed. Under `DAS_TUNE_MODE=tune` that is + the whole `[tune_perm]` grid (gemv and tile, since the gemv is the tile's `tune_companion`); + under normal mode the stamped winner alone. `--perm ` narrows the race (on zen2 the two + VNNI seats decline and run the reference body - visible as reference-speed rows). Timing only: + a spelling's correctness is `DAS_TUNE_MODE=test harness/gen_tune_probe.das`. + ``` + DAS_TUNE_MODE=tune bin/Release/daslang.exe -jit modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das -- --fmt iq3s --perm maddubs --ntok 0 + ``` +2. **The reference row, isolated the same way.** llama.cpp's `test-backend-ops perf -o MUL_MAT` + times one mul_mat per quant type; its default case is m=4096 rows, k=14336, n in + {1,2,3,4,5,8,512} tokens - the microbench's default shape. Thread parity through the + `GGML_BENCH_THREADS` define the harness patch carries (re-applied by hand at the reference tip + 6c84c7d5d; `harness/backend_ops_shapes.patch` no longer applies there): + ``` + GGML_BENCH_THREADS=1 D:/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe perf -b CPU -o MUL_MAT -p "type_a=iq3_s,type_b=f32,m=4096,n=1," + D:/Work/llama.cpp/build-vulkan/bin/Release/test-backend-ops.exe perf -b Vulkan0 -o MUL_MAT -p "type_a=q4_K" + ``` + Vulkan has the same rig on our side in `harness/vk_gemm_probe.das` (`ref` arm = llama.cpp's own + coopmat2 SPV blob inside our harness; `k6x flat` = compose without scale reads). +3. **Outer loop: the app, only when a spelling wins rig 1.** `-jit -module-cache + .jitted_scripts/module_cache/.dascache .das -- --jit-split-modules=-1` (5 s warm, + 65 s after a kernel edit); `--tune-only ` re-crowns one family; `DAS_TUNE_POLICY=reference` + is the A/B baseline. The stock monolith invocation runs once per landed kernel for the table + row. `rm -rf .jitted_scripts` after an emitter edit (`dasllama_gemm_gen.das`) - QUIRK 21. + +Rules that ride along: bench at `DAS_JOBQUE_THREADS=16`, tests at 8; one row at a time; one +proven fact per push; research before any kernel edit (the two memos below). + +## 2. The fact base (zen2 3990X, one thread, m=4096 k=14336, us per run; ours = `dot_maddubs_width256_mr8`) + +| format | ours gemv | llama.cpp n=1 | ours/theirs | ours tile n=512 | llama.cpp n=512 | +|---|---|---|---|---|---| +| k4 | 1969 (8.6 ns/sb) | 2421 | 1.23x | 120.9 GFLOP/s | 71.1 GFLOP/s (1.70x) | +| k2 | 2017 (8.8 ns/sb) | 2132 | 1.06x | - | - | +| iq3s | 11406 (49.7 ns/sb); sign=vec 7562 (33.0) | 10340 | 0.91x -> 1.37x | - | - | +| iq3xxs | 11578 (50.5); sign column + column read 6641 (29.0) | 6590 | 0.57x -> 0.99x | - | - | +| iq2s | 11732 (51.1); sign column + u64 pair 5039-5202 | 5074 | 0.43x -> 0.98-1.01x | - | - | +| iq2xs | 11490 (50.1); sign column + u64 pair + column read 4831 (21.1) | 5386 | 0.47x -> 1.11x | - | - | +| iq2xxs | 11061 (48.2); sign column + u64 pair 5121-5487 (noise band) | 5124 | 0.46x -> 0.93-1.00x | - | - | + +The zen2 ladder after the k3/k2 step and the bench fixes (64-byte-aligned planes, normal scale bytes; +`harness/kernel_ladder.sh`, 2026-09-01, one thread, the stamped seat per format, best of 5 interleaved +rounds; ratio = reference / ours, >= 1.00 = ours faster): + +| format | gemv ours us | gemv ref us | ratio | tile ours us | tile ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 3179 | 4325.63 | 1.36 | 576089 | 923211.00 | 1.60 | +| k4 | 1948 | 2391.08 | 1.23 | 547996 | 821706.50 | 1.50 | +| k5 | 3700 | 3264.03 | 0.88 | 583516 | 1311611.00 | 2.25 | +| k6 | 4204 | 3647.44 | 0.87 | 679148 | 1083505.00 | 1.60 | +| q40 | 1865 | 2839.84 | 1.52 | 387002 | 1018999.00 | 2.63 | +| q51 | 2824 | 4152.78 | 1.47 | 677326 | 1915920.00 | 2.83 | +| iq4xs | 2101 | 3357.24 | 1.60 | 671645 | 1645997.00 | 2.45 | +| k3 | 3092 | 2630.86 | 0.85 | 766074 | 1284702.00 | 1.68 | +| iq3s | 7668 | 10103.66 | 1.32 | 673687 | 5115105.00 | 7.59 | +| iq3xxs | 6601 | 6547.02 | 0.99 | 679775 | 3281595.00 | 4.83 | +| iq4nl | 2208 | 3027.06 | 1.37 | 592394 | 1125061.00 | 1.90 | +| k2 | 1370 | 1973.20 | 1.44 | 692253 | 770237.50 | 1.11 | +| iq2s | 5052 | 5069.15 | 1.00 | 711298 | 2513290.00 | 3.53 | +| iq2xs | 4506 | 5265.71 | 1.17 | 724580 | 2738442.00 | 3.78 | +| iq2xxs | 4859 | 5119.28 | 1.05 | 659785 | 2542052.00 | 3.85 | +| mx4 | 2205 | 3009.07 | 1.36 | 623427 | 1421391.00 | 2.28 | + +k3 decode is the one row that disagrees with its own history (2445 us after its fix, 3092 here) - a +run-to-run spread like Intel's bimodal k6, under investigation. + +The first zen2 ladder (before the k3/k2 step, unaligned planes, random scales): + +| format | gemv ours us | gemv ref us | ratio | tile ours us | tile ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 3324 | 4186.70 | 1.26 | 698131 | 896060.00 | 1.28 | +| k4 | 2089 | 2539.11 | 1.22 | 493006 | 830068.50 | 1.68 | +| k5 | 3628 | 3286.40 | 0.91 | 512129 | 1344947.00 | 2.63 | +| k6 | 4203 | 3785.47 | 0.90 | 640544 | 1080212.00 | 1.69 | +| q40 | 1928 | 2918.57 | 1.51 | 380774 | 1004920.00 | 2.64 | +| q51 | 2816 | 4208.36 | 1.49 | 664024 | 1843932.00 | 2.78 | +| iq4xs | 2100 | 3396.72 | 1.62 | 721313 | 1682846.00 | 2.33 | +| k3 | 3317 | 2638.26 | 0.80 | 775632 | 1267418.00 | 1.63 | +| iq3s | 7671 | 10154.70 | 1.32 | 738713 | 5220515.00 | 7.07 | +| iq3xxs | 6622 | 6564.89 | 0.99 | 741832 | 3395150.00 | 4.58 | +| iq4nl | 2120 | 3134.77 | 1.48 | 599955 | 1076032.00 | 1.79 | +| k2 | 2146 | 2120.78 | 0.99 | 670297 | 783419.00 | 1.17 | +| iq2s | 5059 | 5083.57 | 1.00 | 790327 | 2545281.00 | 3.22 | +| iq2xs | 4554 | 5372.16 | 1.18 | 778490 | 2734115.00 | 3.51 | +| iq2xxs | 5143 | 5157.27 | 1.00 | 745463 | 2534210.00 | 3.40 | +| mx4 | 2572 | 3106.12 | 1.21 | 776635 | 1443088.00 | 1.86 | + +After the k3/k2 step (c720aea95): k3 decode 2445 us (1.08-1.13x), k2 1286-1423 (1.37-1.65x). + +The M1 ladder (Apple M1 Max, one thread, the sdot lattice at width 128, tip b88fc4100; ratio = +reference / ours): decode q8 1.46, k4 1.60, k5 1.06, k6 1.35, q40 1.96, q51 1.32, iq4xs 1.71, k3 +1.37, iq4nl 1.82, k2 1.42, mx4 1.50, iq2s 1.00, iq3s 0.93, iq3xxs 0.72, iq2xxs 0.57, iq2xs 0.51; +every tile 1.9x-15.3x ahead. On ARM the grid formats are the tails - the reference exe's NEON grid +kernels run iq2xs/iq2xxs at 3.3-3.4 ms against our 6.0-6.4 ms while the k-quants are 1.06x-1.60x. + +The zen4 ladder (AWS c7a.4xlarge, EPYC 9R14, one thread, the x86-vnni512 profile's seats = +dot_vpdpbusd_width512_mr16; gen_tune_probe TEST 65/65 ok on the first run of those seats): decode q40 +3.87, iq4nl 2.82, q51 2.65, mx4 2.51, k4 2.25, iq4xs 2.13, k2 2.04, k3 1.88, q8 1.70, k5 1.28, k6 0.91, +iq3xxs 0.88, iq3s 0.81, iq2xxs 0.81, iq2xs 0.79, iq2s 0.77; every tile 2.1x-12x ahead. Where the dot is +the work, the 512-bit VNNI seat pays (k4 1.22x on zen2 -> 2.25x); the five grid formats lose ground +because the reference's grid kernels gain ~1.5x from AVX-512 while ours are bound by the scalar per-dword +gather (iq2s 4898 us on zen4 vs 5059 on zen2) - the ARM memo's diagnosis, on x86. +The Intel ladder (AWS c8i.4xlarge, Xeon 6975P-C Granite Rapids, one thread, the new x86-amx class - +its profile minted here and shipped; TEST 65/65 ok with the AMX leg; the bench with 64-byte-aligned +planes and normal scale bytes): decode q51 5.07, mx4 3.26, q8 3.21, iq4xs 2.99, q40 2.92, iq4nl 2.88, +k3 2.47, k2 2.20, k5 1.94, k4 1.54, iq2xs 1.15, iq3s 1.10, iq2xxs 1.10, iq3xxs 0.94, iq2s 0.91, +k6 0.39 (bimodal on this box: 2.1 ms in two runs, 5.8-6.7 in two - allocation-dependent past 64-byte +alignment, open); every tile 2.06x-14.2x ahead. The reference exe's own rows swung up to 2x between runs +on the VM (q8_0 4036 -> 7487), so single-run Intel ratios carry that error bar. Two bench artifacts were +found and fixed on this box: 16-byte-aligned planes (Intel splits a 64-byte load across a cache line - +k6 6703 -> 2234 us) and random scale bytes (denormal math - q8 tile 1022 -> 155 ms). Open: the bench's +tune-mode q8 tile SIGILLs on the AMX box even after invoking the witness variants; the tuner's own arming +works (the mint raced the AMX tiles), so the bench arms AMX differently from gen_tune_probe - find how. + +Model level on the zen4 (Q4_K_M 1B, 16 threads, debug-jit rows): pp512 1172 vs 927 (1.26x), tg128 88.7 +vs 86.0 (1.03x). The kernel ladder's k4 2.25x is against the reference's fallback vec_dot; a model run +uses its AVX2/AVX-512 repack 8x8 GEMV for q4_K - so the ladder's k4/k5/k6/q4_0/q8_0/iq4_nl/q2_K rows +overstate against real serving where the reference repacks (repack.cpp covers those eight types); the +grid formats and q3_K have no repack path and their rows are the fair ones. + +zen2 reading: every tile row is ahead (1.17x-7.07x). Decode tails were k3 0.80x, k6 0.90x, k5 0.91x; iq3xxs, k2, iq2s, +iq2xxs at 0.99-1.00 (inside the noise band). The 4-bit class (q40, q51, iq4xs, iq4nl, k4, q8, mx4) +sits at 1.2x-1.6x. + +Model level, zen2 16 threads, 1B vehicles, tg128 ours vs the clean-CPU llama-bench (before the pass): +IQ3_M (iq3s) 69.9 vs 56.5 = 1.24x (0.92x); IQ3_XXS-local 74.9 vs 74.3 = 1.01x (0.78x); IQ2_XS-local +93.8 vs 85.9 = 1.09x (0.70x); IQ2_XXS-local 88.6 vs 86.2 = 1.03x (0.70x); i1-IQ3_XXS (the mixed +iq2s/iq3xxs/iq3s vehicle) 71.9 +-4.2 vs 74.5 = 0.97x (0.76x). pp512 3.6x-5.0x throughout. Debug-jit +rows (`--for-debug-purposes`), 3 reps; the record-grade rows come with the released exe at arc end. + +Reading: the five grid formats cost 48-51 ns per superblock regardless of what each decodes, +against 8.6-8.8 for k4/k2. A flat cost independent of the format is a shared mechanism, not five +decode problems. The model-level tg rows (0.70-0.92x) are this 2x kernel gap hidden behind memory +bandwidth; on the 1B vehicles it shows, on a 27B it mostly does not - but the kernel is the same. + +### zen2 v3 - the arena bench (2026-09-01, `harness/kernel_ladder.sh all`, one thread, best of 5 interleaved rounds) + +Every plane in ONE arena at fixed 64-byte-aligned staggered offsets; the reference is test-backend-ops at +6c84c7d5d with the GGML_BENCH_THREADS define. k6 read 5356 us here against 4.0-4.4 ms in the arena smoke - +its process-to-process swing is narrowed, not gone. + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 3142 / 4220 | 1.34 | 567862 / 907771 | 1.60 | +| k4 | 2005 / 2434 | 1.21 | 485803 / 846691 | 1.74 | +| k5 | 3467 / 3306 | 0.95 | 514648 / 1296856 | 2.52 | +| k6 | 5356 / 3683 | 0.69 | 634588 / 1076847 | 1.70 | +| q40 | 1901 / 2971 | 1.56 | 374388 / 1005240 | 2.69 | +| q51 | 2815 / 4107 | 1.46 | 662669 / 1830207 | 2.76 | +| iq4xs | 2076 / 3375 | 1.63 | 723720 / 1684220 | 2.33 | +| k3 | 2699 / 2700 | 1.00 | 733239 / 1245200 | 1.70 | +| iq3s | 7593 / 10345 | 1.36 | 743364 / 5194190 | 6.99 | +| iq3xxs | 6554 / 6524 | 1.00 | 739558 / 3293421 | 4.45 | +| iq4nl | 2126 / 3055 | 1.44 | 581628 / 1094468 | 1.88 | +| k2 | 1248 / 2158 | 1.73 | 636260 / 764748 | 1.20 | +| iq2s | 5044 / 5089 | 1.01 | 778057 / 2512394 | 3.23 | +| iq2xs | 4709 / 5320 | 1.13 | 799146 / 2692158 | 3.37 | +| iq2xxs | 4897 / 5086 | 1.04 | 738382 / 2537062 | 3.44 | +| mx4 | 2216 / 3113 | 1.41 | 618514 / 1441041 | 2.33 | + +Decode below 1.0: k6 0.69, k5 0.95; at 1.0: k3 1.00, iq3xxs 1.00, iq2s 1.01. Every tile ahead (1.20-6.99). + +M1 note: the ladder's reference must be `~/Work/llama.cpp/build/bin/test-backend-ops` (the fork checkout, +98c4764b6, carries the define); `build-cpu/` does not and ran every core - the ladder now refuses such a binary. + +### M1 v3 - the full ladder after the ARM grid landing (2026-09-01, one thread, best of 5 interleaved rounds) + +`~/Work/llama.cpp/build/bin/test-backend-ops` (the fork checkout at 98c4764b6, the define in). EVERY row at or +above parity - the M1 CPU is closed for this pass. + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 1059 / 1562 | 1.48 | 214129 / 434961 | 2.03 | +| k4 | 839 / 1343 | 1.60 | 268955 / 703992 | 2.62 | +| k5 | 2067 / 2185 | 1.06 | 252434 / 1129322 | 4.47 | +| k6 | 1777 / 2401 | 1.35 | 401658 / 1243030 | 3.09 | +| q40 | 863 / 1612 | 1.87 | 276856 / 531679 | 1.92 | +| q51 | 2075 / 2749 | 1.33 | 453329 / 1436227 | 3.17 | +| iq4xs | 988 / 1680 | 1.70 | 262229 / 870313 | 3.32 | +| k3 | 1925 / 2619 | 1.36 | 486461 / 1348956 | 2.77 | +| iq3s | 4914 / 5680 | 1.16 | 192154 / 2922617 | 15.21 | +| iq3xxs | 3572 / 4902 | 1.37 | 192186 / 2492096 | 12.97 | +| iq4nl | 1008 / 1847 | 1.83 | 290154 / 949701 | 3.27 | +| k2 | 1348 / 1915 | 1.42 | 357321 / 986778 | 2.76 | +| iq2s | 3259 / 4915 | 1.51 | 255732 / 2516761 | 9.84 | +| iq2xs | 2635 / 3249 | 1.23 | 255582 / 1669316 | 6.53 | +| iq2xxs | 2871 / 3435 | 1.20 | 192075 / 1765362 | 9.19 | +| mx4 | 1271 / 1900 | 1.50 | 331361 / 978109 | 2.95 | + +Decode floor k5 1.06x; the grid formats 1.16-1.51x (from 0.51-0.93x before the row-pair decode). Tiles 1.92-15.2x. + +### M1 v4 - closing tables (2026-09-01, after the k6/k3 transposes; one thread at the engine phase, and the engine shape: 8 lanes, d=32768) + +| fmt | decode ours / ref us | ratio | engine 8 lanes ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 1059 / 1537 | 1.45 | 4187 / 4519 | 1.08 | +| k4 | 840 / 1343 | 1.60 | 2240 / 2271 | 1.01 | +| k5 | 2068 / 2184 | 1.06 | 2757 / 2715 | 0.98 | +| k6 | 1699 / 2405 | 1.42 | 3224 / 3284 | 1.02 | +| q40 | 768 / 1623 | 2.11 | 2176 / 2231 | 1.03 | +| q51 | 2075 / 2752 | 1.33 | 2922 / 3145 | 1.08 | +| iq4xs | 987 / 1699 | 1.72 | 2234 / 2125 | 0.95 | +| k3 | 1876 / 2621 | 1.40 | 2016 / 2831 | 1.40 | +| iq3s | 4924 / 5714 | 1.16 | 5282 / 6096 | 1.15 | +| iq3xxs | 3813 / 4876 | 1.28 | 3949 / 5208 | 1.32 | +| iq4nl | 1036 / 1828 | 1.76 | 2175 / 2272 | 1.04 | +| k2 | 1390 / 1918 | 1.38 | 1463 / 2138 | 1.46 | +| iq2s | 3255 / 4932 | 1.52 | 3669 / 5366 | 1.46 | +| iq2xs | 2637 / 3261 | 1.24 | 2835 / 3490 | 1.23 | +| iq2xxs | 2913 / 3440 | 1.18 | 3112 / 3718 | 1.19 | +| mx4 | 1272 / 1910 | 1.50 | 1990 / 2147 | 1.08 | + +One thread: every decode >= 1.06 (k5 the floor; k6 1.42, k3 1.40). Engine shape: every format >= 0.95 (iq4xs), +most 1.0-1.5. The M1 CPU is closed at both shapes. + +### Intel v2 - Granite Rapids (2026-09-01, c8i.4xlarge Xeon 6975P-C, class x86-amx, branch 095f4030c; TEST 90/90) + +The bootstrap's two ladders both ran the new default (engine shape: 16 lanes, d=32768) - the one-thread table and +the knob A/B run in round two. + +| fmt | engine 16 lanes ours / ref us | ratio | +|---|---|---| +| q8 | 4539 / 4835 | 1.07 | +| k4 | 2332 / 2849 | 1.22 | +| k5 | 3073 / 3862 | 1.26 | +| k6 | 5230 / 3587 | 0.69 | +| q40 | 2345 / 4923 | 2.10 | +| q51 | 3196 / 5238 | 1.64 | +| iq4xs | 2367 / 4355 | 1.84 | +| k3 | 2703 / 2579 | 0.95 | +| iq3s | 7310 / 7575 | 1.04 | +| iq3xxs | 5396 / 6121 | 1.13 | +| iq4nl | 2359 / 5928 | 2.51 | +| k2 | 1229 / 1598 | 1.30 | +| iq2s | 4283 / 3786 | 0.88 | +| iq2xs | 3962 / 4579 | 1.16 | +| iq2xxs | 3805 / 3853 | 1.01 | +| mx4 | 2467 / 5802 | 2.35 | + +The reference does not reach this box's DRAM on most formats (q4_0 4.9 ms for 264 MB = 54 GB/s) where ours does +(2.3 ms = 115 GB/s) - hence 2.1-2.5x on q40/iq4nl/mx4. Tails: k6 0.69 (5.2 ms against 3.5 - the reference +saturates DRAM on q6_K, our k6 stays compute-bound; the vpdpbusd512 seat has no i16 chain for the fused flush), +iq2s 0.88, k3 0.95. + +Round two, one thread at the engine phase (branch eb94f1d9a): + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 2224 / 2687 | 1.21 | 145999 / 578161 | 3.96 | +| k4 | 1140 / 1748 | 1.53 | 263222 / 787687 | 2.99 | +| k5 | 1525 / 2447 | 1.60 | 175535 / 1133333 | 6.46 | +| k6 | 1840 / 2040 | 1.11 | 239252 / 834122 | 3.49 | +| q40 | 1108 / 2288 | 2.06 | 175513 / 651009 | 3.71 | +| q51 | 1488 / 2946 | 1.98 | 214319 / 1410780 | 6.58 | +| iq4xs | 1126 / 2380 | 2.11 | 272987 / 1144376 | 4.19 | +| k3 | 1277 / 2266 | 1.77 | 331164 / 1115864 | 3.37 | +| iq3s | 5207 / 6783 | 1.30 | 252732 / 3499939 | 13.85 | +| iq3xxs | 4505 / 5053 | 1.12 | 252332 / 2480068 | 9.83 | +| iq4nl | 1122 / 2686 | 2.39 | 263188 / 717486 | 2.73 | +| k2 | 716 / 1468 | 2.05 | 356125 / 688367 | 1.93 | +| iq2s | 3996 / 3709 | 0.93 | 272607 / 1851295 | 6.79 | +| iq2xs | 3955 / 4342 | 1.10 | 272793 / 2184555 | 8.01 | +| iq2xxs | 3675 / 3760 | 1.02 | 252192 / 1799256 | 7.13 | +| mx4 | 1285 / 2990 | 2.33 | 273869 / 1247422 | 4.55 | + +One thread: every decode >= 1.02 except iq2s 0.93; k6 1.11. Tiles 1.9-13.9x. +The seat races (tune mode, every registry row; us): k6 one thread maddubs256 1648 / vpdpbusd256 1636 / +vpdpbusd512 1615 (reference 1952), at 16 lanes d=32768 3489 / 3424 / 3419 against the reference's 3587 - +0.98-1.05x, so the bootstrap ladder's k6 0.69 was that first run's artifact, not the kernel (round three +re-measures it cold). k3: 1246 / 1125 / 922 alone (reference 2258), 1780 / 1797 / 1525 at 16 lanes +(2579). iq2s: 4060 / 3855 / 3957 alone (3702), 4242 / 4090 / 4208 at 16 lanes (3786) - the crowned width-512 +seat loses 3% to vpdpbusd256 at the engine shape, as on zen4: the gemv inherits the tile's crown +(tune_companion), and the tile and the gemv do not agree on width. The knob A/B (the table-sign row form): +iq2s 5345, iq2xs 5033, iq2xxs 3754-3820, iq3s 9763, iq3xxs 5606 against the panel's 3956 / 3902 / 3904 / +5276 / 4601 - the panel everywhere but iq2xxs (marginal); the column-sign form is round three's question. +Round three (f68534c06, the column-sign row form), one thread panel -> rows: iq2xxs 3678 -> 2518 (reference 3708: +1.47x), iq3xxs 4504 -> 4392 (5092: 1.16x); iq2s 3994 -> 5428, iq2xs 3962 -> 5504, iq3s 5209 -> 9785 stay panel. +At 16 lanes: iq2xxs 3811 -> 2614 (3853: 1.47x), iq3xxs 5448 -> 4515 (6121: 1.36x), the other three worse. Eight +lanes (one per core) against 16: k6 6562 -> 3420-5219, k3 3720 -> 2723, iq2s 4431 -> 4287, iq3s 9278 -> 7339 - +more lanes help every format here, 16 stands. LANDED (d9928f985): the x86-amx class takes the row form for iq2xxs +and iq3xxs; the default verified on the box - TEST 90/90, iq2xxs 2473-2496 alone / 2606 at 16 lanes (1.49x / +1.48x), iq3xxs 4390 / 4531 (1.16x / 1.35x), iq2s 3965-3991 / 4280 (0.93 / 0.88), iq2xs 3927-3973 / 3926 (1.10 / +1.17), iq3s 5081-5182 / 7254 (1.30 / 1.05), k6 at 16 lanes 5316 in normal mode against 3419-3489 for the same +stamped seat in the tune-mode race - a mode difference, not the kernel (the back-to-back probe runs next). +The probe: same 16 lanes, same 128 x 256-row split, same seat - 5270 us when its calls run back to back (normal +mode), 3284-3448 when interleaved with ~60 ms of scalar rows (tune mode), three times over. That is the shape of +Intel's AVX-512 frequency license (sustained 512-bit work drops the core clock; the reference's q6_K runs 256-bit); +the seat-alone runs (each vector seat back to back at 16 lanes) decide it, and the mint's tile race (bursty per +seat) may have crowned width 512 where the sustained gemv wants 256 - the gemv-own-crown item once more. +The seat-alone runs said no to the license: 512 alone 3713, 256 alone 3551, maddubs alone 3541 (reference 3441) - +back to back, all near parity - so it is the MODE: the same seat generated under the adopted profile (normal mode) +runs 5159-5270 at 16 lanes and 1861-2244 / 1939-3622 (bimodal within a process) at one thread, where the tune-mode +race of the same name reads 3284-3713 and 1615-1789 / 1648-2059. zen2 shows no such gap (normal 3446-4033, tune +3657-4076). The 64-byte frame realignment (e0b3892b7) did not move it. The JIT'd body is emitted at runtime and is +not in the cache, so the two bodies could not be diffed here. OPEN, on the Intel box: whether the adopted perm +parses to the grid row's fields (defaults) or the runtime differs; the gemv's own in-situ race (queued) measures +every seat the same way and stamps what is fastest, which closes it either way. + + +### M4 Pro v1 - first tables (2026-09-01, Apple M4 Pro 10P+4E, class arm-i8mm, on the arm-neon PROFILE through the chain - no arm-i8mm profile existed; TEST 90/90) + +One thread at the engine phase; the engine shape = 10 performance cores, d=32768. The SMMLA lattice's first real race is +the tile column. + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | engine 10 lanes ours / ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 693 / 1125 | 1.62 | 184237 / 425873 | 2.31 | 2179 / 2078 | 0.95 | +| k4 | 649 / 916 | 1.41 | 210798 / 458459 | 2.17 | 1107 / 1132 | 1.02 | +| k5 | 1625 / 1464 | 0.90 | 201095 / 766460 | 3.81 | 1556 / 1521 | 0.98 | +| k6 | 1304 / 1731 | 1.33 | 329965 / 732022 | 2.22 | 1611 / 1824 | 1.13 | +| q40 | 596 / 1262 | 2.12 | 218731 / 516770 | 2.36 | 1055 / 1342 | 1.27 | +| q51 | 1637 / 1996 | 1.22 | 356194 / 1035551 | 2.91 | 1445 / 2306 | 1.60 | +| iq4xs | 780 / 1308 | 1.68 | 208597 / 670258 | 3.21 | 1091 / 1269 | 1.16 | +| k3 | 1435 / 1824 | 1.27 | 368506 / 955375 | 2.59 | 1257 / 1864 | 1.48 | +| iq3s | 3904 / 4061 | 1.04 | 155531 / 2102599 | 13.52 | 3298 / 3841 | 1.16 | +| iq3xxs | 2661 / 3755 | 1.41 | 155576 / 1927299 | 12.39 | 2263 / 3539 | 1.56 | +| iq4nl | 788 / 1430 | 1.81 | 231408 / 734682 | 3.17 | 1047 / 1387 | 1.32 | +| k2 | 969 / 1395 | 1.44 | 279559 / 710512 | 2.54 | 854 / 1302 | 1.52 | +| iq2s | 2386 / 2975 | 1.25 | 183660 / 1504690 | 8.19 | 2233 / 2572 | 1.15 | +| iq2xs | 1786 / 2435 | 1.36 | 184119 / 1250945 | 6.79 | 1530 / 2240 | 1.46 | +| iq2xxs | 2226 / 2623 | 1.18 | 155669 / 1336029 | 8.58 | 1892 / 2328 | 1.23 | +| mx4 | 966 / 1322 | 1.37 | 263140 / 679457 | 2.58 | 982 / 1521 | 1.55 | + +One thread: every decode >= 1.04 except k5 0.90; tiles 2.2-13.5x. Engine shape: everything >= 0.95 (q8), k5 0.98, +the rest 1.02-1.60. The arm-i8mm mint (the 1B vehicle, --tune, export) runs next on this box and the tables re-run +on the minted profile. +MINTED (fd8e490f1: provenance ok / ok / neon;dotprod;i8mm;bf16;fullfp16;lse, 49 kernels): the kq tile crowns are +the neon profile's (`mr8` everywhere), q8q8_tile_gen moves mr8_budget -> kstep2, seven [tuned] loop kernels pick +other widths. One thread on the minted profile, us (reference): q8 727 (1128) 1.55, k4 666 (916) 1.38, k5 1643 +(1466) 0.89, k6 1312 (1732) 1.32, q40 605 (1263) 2.09, q51 1651 (1994) 1.21, iq4xs 815 (1311) 1.61, k3 1454 (1829) +1.26, iq3s 3888 (4066) 1.05, iq3xxs 2678 (3762) 1.40, iq4nl 795 (1433) 1.80, k2 976 (1404) 1.44, iq2s 2383 (2952) +1.24, iq2xs 1787 (2441) 1.37, iq2xxs 2224 (2616) 1.18, mx4 995 (1322) 1.33 - the same table as before, as the +crowns say. k5 0.89 is the M4's one tail (the M1 has it at 1.06; the P-core's byte->lane deposit costs more). +Engine shape on the minted profile (10 P-cores, d=32768), ours / reference us: q8 2178 / 2082 (0.96), k4 1116 / +1146 (1.03), k5 1549 / 1546 (1.00), k6 1627 / 1843 (1.13), q40 1057 / 1368 (1.29), q51 1447 / 2344 (1.62), iq4xs +1120 / 1292 (1.15), k3 1244 / 1909 (1.53), iq3s 3308 / 4059 (1.23), iq3xxs 2258 / 3735 (1.65), iq4nl 1055 / 1451 +(1.38), k2 850 / 1343 (1.58), iq2s 2242 / 2624 (1.17), iq2xs 1698 / 2343 (1.38), iq2xxs 1894 / 2416 (1.28), mx4 +983 / 1553 (1.58). The M4 CPU is at or above parity on every engine-shape row; k5 0.89 alone was the one tail. +LANDED: the NEON per-lane byte shift for k5's high-bit deposit (`ushl` / `ushr` + `and` per half, where the mask +test took five ops; sdot lattice only - x86 has no byte shift): M4 k5 decode 1625-1643 -> 1357 us (reference +1466: 1.08x), engine shape 1549 -> 1360 (1.14x), tile unchanged, k4 unchanged; TEST 90/90 on the M4 and zen2. +The M4 is closed on every row at both shapes. + +### zen2 v4 - the closing tables (2026-09-01, after the k6/k3 transposes and the fused flush) + +One thread at the HEAP phase (the ladder's engine-phase default came after this run; k6/k3 read 10-30% +better at the engine phase - k6 3527-3851, k3 2488-2893) and the engine shape (16 lanes, d=32768, +DRAM-bound; q8/mx4/k4 from the q8-family arm's validation run). + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | engine 16 lanes ours / ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 3348 / 4260 | 1.27 | 565887 / 890979 | 1.57 | 8906 / 9248 | 1.04 | +| k4 | 2337 / 2528 | 1.08 | 518533 / 839960 | 1.62 | 4472 / 4425 | 0.99 | +| k5 | 3553 / 3327 | 0.94 | 540857 / 1325657 | 2.45 | 5400 / 5475 | 1.01 | +| k6 | 3460 / 3652 | 1.06 | 630854 / 1083547 | 1.72 | 6477 / 6402 | 0.99 | +| q40 | 1973 / 2868 | 1.45 | 387611 / 1003869 | 2.59 | 4275 / 4575 | 1.07 | +| q51 | 2866 / 4118 | 1.44 | 656416 / 1844397 | 2.81 | 5842 / 6235 | 1.07 | +| iq4xs | 2097 / 3349 | 1.60 | 675053 / 1674064 | 2.48 | 4577 / 4119 | 0.90 | +| k3 | 3080 / 2601 | 0.84 | 678487 / 1255396 | 1.85 | 3662 / 3605 | 0.98 | +| iq3s | 7603 / 10242 | 1.35 | 660738 / 5179767 | 7.84 | 3934 / 5300 | 1.35 | +| iq3xxs | 6650 / 6573 | 0.99 | 659839 / 3308544 | 5.01 | 3854 / 3509 | 0.91 | +| iq4nl | 2117 / 3078 | 1.45 | 589114 / 1090653 | 1.85 | 4542 / 4533 | 1.00 | +| k2 | 1261 / 2083 | 1.65 | 658936 / 779599 | 1.18 | 2555 / 2711 | 1.06 | +| iq2s | 5058 / 5022 | 0.99 | 721269 / 2510375 | 3.48 | 3070 / 2887 | 0.94 | +| iq2xs | 4645 / 5354 | 1.15 | 735654 / 2706628 | 3.68 | 2749 / 2886 | 1.05 | +| iq2xxs | 5192 / 5103 | 0.98 | 655125 / 2526484 | 3.86 | 2994 / 2751 | 0.92 | +| mx4 | 2193 / 3004 | 1.37 | 621096 / 1433680 | 2.31 | 4143 / 4364 | 1.05 | + +Engine shape: every format 0.90-1.35 - q8 1.04, mx4 1.05, k4 0.99, k5 1.01, k6 0.99, q40 1.07, q51 1.07, +k3 0.98, iq3s 1.35, iq4nl 1.00, k2 1.06, iq2xs 1.05, iq2s 0.94, iq2xxs 0.92, iq3xxs 0.91, iq4xs 0.90. zen2 +is closed for this pass: k5 0.94 at one thread is the one row under the bar, and it is 1.01 at the engine shape. + +### zen4 v2 - after the transposes and the fused flush (2026-09-01, c7a.4xlarge EPYC 9R14, class x86-vnni512) + +TEST 90/90. One thread at the heap phase (the bootstrap ran before the ladder's engine-phase default) and the +engine shape (16 lanes, d=32768, DRAM-bound). The q8/mx4 engine rows here predate the q8 family's team arm +(one raw thread against 16) - round two re-measures them. + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | engine 16 lanes ours / ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 1614 / 3409 | 2.11 | 167078 / 500478 | 3.00 | 15344 / 6101 | 0.40 | +| k4 | 805 / 1865 | 2.32 | 328569 / 753160 | 2.29 | 3954 / 3028 | 0.77 | +| k5 | 2126 / 2615 | 1.23 | 178594 / 1120210 | 6.27 | 3788 / 3802 | 1.00 | +| k6 | 3045 / 2872 | 0.94 | 226619 / 782510 | 3.45 | 4840 / 4609 | 0.95 | +| q40 | 653 / 2743 | 4.20 | 172935 / 588322 | 3.40 | 3028 / 2963 | 0.98 | +| q51 | 1864 / 4607 | 2.47 | 239601 / 1431282 | 5.97 | 4149 / 4165 | 1.00 | +| iq4xs | 1180 / 2418 | 2.05 | 286066 / 1153787 | 4.03 | 2809 / 2848 | 1.01 | +| k3 | 833 / 2031 | 2.44 | 362356 / 978906 | 2.70 | 2318 / 2207 | 0.95 | +| iq3s | 7527 / 6087 | 0.81 | 259660 / 3089555 | 11.90 | 3903 / 3226 | 0.83 | +| iq3xxs | 5953 / 5234 | 0.88 | 260765 / 2664217 | 10.22 | 3122 / 2839 | 0.91 | +| iq4nl | 1066 / 3103 | 2.91 | 283855 / 625809 | 2.20 | 3084 / 3026 | 0.98 | +| k2 | 630 / 1276 | 2.03 | 429374 / 641108 | 1.49 | 1541 / 1605 | 1.04 | +| iq2s | 4917 / 3748 | 0.76 | 283651 / 1903473 | 6.71 | 2778 / 2116 | 0.76 | +| iq2xs | 4549 / 3616 | 0.80 | 283584 / 1844309 | 6.50 | 2312 / 2001 | 0.87 | +| iq2xxs | 4834 / 3915 | 0.81 | 260122 / 1991565 | 7.66 | 2464 / 2238 | 0.91 | +| mx4 | 1049 / 2740 | 2.61 | 293620 / 1348460 | 4.59 | 9599 / 2849 | 0.30 | + +One thread: every non-grid decode 0.94-4.20x (k6 0.94), tiles 1.5-11.9x; the grid formats 0.76-0.88x - +the x86 tail, unchanged. Engine shape: k5/q51/iq4xs/k2 1.00-1.04, q40/iq4nl 0.98, k6/k3 0.95, iq3xxs/iq2xxs +0.91, iq2xs 0.87, iq3s 0.83, iq2s 0.76 - and k4 0.77 (67 GB/s where the reference and our k5 pull 86; 2.32x +at one thread). Round two on the same box: SOLO at the engine phase, the corrected team rows, a tune-mode race +of k4/k6/iq2s through the team arm, perf stat on iq2s. + +Round two (the branch at 6d75a0956): + +| fmt | one thread, engine phase ours / ref us | ratio | engine 16 lanes ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 1874 / 3657 | 1.95 | 6575 / 6058 | 0.92 | +| k4 | 833 / 1906 | 2.29 | 3067 / 3022 | 0.99 | +| k5 | 2046 / 2885 | 1.41 | 4043 / 3794 | 0.94 | +| k6 | 2866 / 2686 | 0.94 | 4861 / 4600 | 0.95 | +| q40 | 737 / 2794 | 3.79 | 3163 / 2996 | 0.95 | +| q51 | 1690 / 4630 | 2.74 | 3940 / 4172 | 1.06 | +| iq4xs | 1176 / 2438 | 2.07 | 3036 / 2834 | 0.93 | +| k3 | 837 / 2016 | 2.41 | 2268 / 2209 | 0.97 | +| iq3s | 7571 / 6104 | 0.81 | 4128 / 3213 | 0.78 | +| iq3xxs | 5973 / 5242 | 0.88 | 3113 / 2802 | 0.90 | +| iq4nl | 1092 / 3055 | 2.80 | 2915 / 3025 | 1.04 | +| k2 | 623 / 1289 | 2.07 | 1461 / 1596 | 1.09 | +| iq2s | 4918 / 3763 | 0.77 | 2525 / 2129 | 0.84 | +| iq2xs | 4562 / 3633 | 0.80 | 2319 / 2001 | 0.86 | +| iq2xxs | 4857 / 3920 | 0.81 | 2461 / 2127 | 0.86 | +| mx4 | 1090 / 2763 | 2.54 | 2787 / 2844 | 1.02 | + +Engine shape: q51 1.06, iq4nl 1.04, k2 1.09, mx4 1.02, k4 0.99 (the 0.77 above was a one-off), k3 0.97, k6 0.95, +q40 0.95, k5 0.94, iq4xs 0.93, q8 0.92 - and the grid formats iq3xxs 0.90, iq2xs/iq2xxs 0.86, iq2s 0.84, iq3s +0.78: compute-bound even at 16 lanes (iq3s pulls 49 GB/s of the box's 87). The tune race through the team arm +(d=32768, 16 lanes): k4 maddubs256 3084 / vpdpbusd256 3030 / vpdpbusd512 2969 and k6 4630 / 4611 / 4532 confirm +the crowns; iq2s 2528 / 2383 / 2530 - the crowned width-512 seat loses 6% to vpdpbusd256 at the engine shape, +the gemv inheriting the tile's crown (tune_companion). perf stat gave nothing (no PMU in the VM). + +### zen4 v3 - the re-minted x86-vnni512 profile (2026-09-01, c7a.4xlarge EPYC 9R14, commit 11f69493e; `SOLO=1 NTOK=512` one thread at d=4096 for gemv/tile, then the team ladder: 16 lanes, d=32768, best of 5) + +| fmt | gemv 1T ours / ref | ratio | tile 1T ours / ref | ratio | gemv 16 lanes ours / ref | ratio | +|---|---|---|---|---|---|---| +| q8 | 1983 / 3650 | 1.84 | 167509 / 501447 | 2.99 | 6671 / 5995 | 0.90 | +| k4 | 843 / 1878 | 2.23 | 350323 / 753000 | 2.15 | 2914 / 3049 | 1.05 | +| k5 | 2160 / 2685 | 1.24 | 178164 / 1124541 | 6.31 | 3873 / 3824 | 0.99 | +| k6 | 2478 / 2992 | 1.21 | 226354 / 787084 | 3.48 | 4637 / 4682 | 1.01 | +| q40 | 693 / 2726 | 3.93 | 173654 / 588905 | 3.39 | 3027 / 2940 | 0.97 | +| q51 | 1559 / 4640 | 2.98 | 285388 / 1428594 | 5.01 | 4327 / 4300 | 0.99 | +| iq4xs | 1129 / 2449 | 2.17 | 286750 / 1155395 | 4.03 | 3225 / 2856 | 0.89 | +| k3 | 942 / 2022 | 2.15 | 355086 / 979214 | 2.76 | 2212 / 2161 | 0.98 | +| iq3s | 7556 / 6092 | 0.81 | 261572 / 3093452 | 11.83 | 3896 / 3260 | 0.84 | +| iq3xxs | 5964 / 5226 | 0.88 | 260290 / 2660673 | 10.22 | 3176 / 2818 | 0.89 | +| iq4nl | 1075 / 3083 | 2.87 | 284054 / 624404 | 2.20 | 3115 / 2882 | 0.93 | +| k2 | 622 / 1281 | 2.06 | 430224 / 641204 | 1.49 | 1433 / 1611 | 1.12 | +| iq2s | 4915 / 3756 | 0.76 | 283819 / 1903444 | 6.71 | 2524 / 2124 | 0.84 | +| iq2xs | 4555 / 3621 | 0.79 | 283489 / 1845848 | 6.51 | 2559 / 2003 | 0.78 | +| iq2xxs | 2854 / 3916 | 1.37 | 260753 / 1992837 | 7.64 | 1657 / 2116 | 1.28 | +| mx4 | 1059 / 2746 | 2.59 | 292052 / 1349795 | 4.62 | 2895 / 2864 | 0.99 | + +Reading: one thread, everything is past the reference except the four panel-form grids (iq3s 0.81, iq3xxs +0.88, iq2s 0.76, iq2xs 0.79; iq2xxs on the row form 1.37) - the compute-bound gap the zen4 memo targets. At +16 lanes the box is DRAM-bound (k4 ours 235 MB / 2.9 ms = 81 GB/s, the reference 77): the 0.89-0.99 rows +there are within the shared box's bandwidth noise, the grids repeat their one-thread losses (0.78-0.89). + +### Intel v3 - the re-minted x86-amx profile (2026-09-01, c8i.4xlarge Xeon 6975P-C, commit 11f69493e; same ladders) + +| fmt | gemv 1T ours / ref | ratio | tile 1T ours / ref | ratio | gemv 16 lanes ours / ref | ratio | +|---|---|---|---|---|---|---| +| q8 | 2226 / 2813 | 1.26 | 153424 / 578235 | 3.77 | 4414 / 4319 | 0.98 | +| k4 | 1138 / 1704 | 1.50 | 234490 / 847640 | 3.61 | 2057 / 3083 | 1.50 | +| k5 | 1567 / 2471 | 1.58 | 176873 / 1185485 | 6.70 | 2807 / 4063 | 1.45 | +| k6 | 1831 / 1883 | 1.03 | 218123 / 912830 | 4.18 | 5227 / 4353 | 0.83 | +| q40 | 1107 / 2289 | 2.07 | 168018 / 649524 | 3.87 | 2065 / 5143 | 2.49 | +| q51 | 1472 / 2915 | 1.98 | 227348 / 1462531 | 6.43 | 2907 / 5406 | 1.86 | +| iq4xs | 1109 / 2378 | 2.14 | 280699 / 1158466 | 4.13 | 2169 / 4817 | 2.22 | +| k3 | 1272 / 2269 | 1.78 | 297557 / 1143046 | 3.84 | 2642 / 2687 | 1.02 | +| iq3s | 5152 / 7079 | 1.37 | 251684 / 3550699 | 14.11 | 7426 / 7413 | 1.00 | +| iq3xxs | 4386 / 5073 | 1.16 | 250846 / 2597215 | 10.35 | 4533 / 6704 | 1.48 | +| iq4nl | 1113 / 2710 | 2.44 | 264126 / 714114 | 2.70 | 2079 / 6320 | 3.04 | +| k2 | 660 / 1470 | 2.23 | 287013 / 699910 | 2.44 | 957 / 1729 | 1.81 | +| iq2s | 3963 / 3690 | 0.93 | 275657 / 1885050 | 6.84 | 4266 / 3848 | 0.90 | +| iq2xs | 3914 / 4326 | 1.11 | 275710 / 2206824 | 8.00 | 3931 / 4610 | 1.17 | +| iq2xxs | 2505 / 3666 | 1.46 | 251397 / 1828919 | 7.28 | 2589 / 4255 | 1.64 | +| mx4 | 1207 / 3038 | 2.52 | 275059 / 1311864 | 4.77 | 2488 / 6361 | 2.56 | + +Reading: 16 lanes at or past the reference except k6 0.83 (5227 us - the normal-mode gap, see the queue; +one thread shows it as best 1831 / median 2925 inside ONE process, so the slow mode is the common one and +flips between rounds, not between processes) and iq2s 0.90 (0.93 one thread). No gemv seat was written on +this box, so the k6 gemv still rides the 512 tile crown here. + +### zen4 v4 and Intel v4 - the goal ladders on the lattice re-mints (2026-09-01; SOLO one thread d=4096, TEAM 16 lanes d=32768; ratio = reference / ours) + +| fmt | zen4 gemv 1T | zen4 tile 1T | zen4 gemv 16L | Intel gemv 1T | Intel tile 1T | Intel gemv 16L | +|---|---|---|---|---|---|---| +| q8 | 1.81 | 3.01 | 0.89 | 1.25 | 4.00 | 0.95 | +| k4 | 2.45 | 2.19 | 1.05 | 1.52 | 3.88 | 1.47 | +| k5 | 1.26 | 6.33 | 1.00 | 1.69 | 6.78 | 1.48 | +| k6 | 1.10 | 3.43 | 1.01 | 1.19 | 4.14 | 0.68 | +| q40 | 4.31 | 3.39 | 1.00 | 2.07 | 3.84 | 2.55 | +| q51 | 2.84 | 5.03 | 0.94 | 2.51 | 7.01 | 1.84 | +| iq4xs | 2.13 | 4.03 | 0.92 | 2.14 | 4.26 | 2.32 | +| k3 | 2.18 | 2.75 | 0.99 | 1.85 | 4.11 | 1.01 | +| iq3s | 2.20 | 11.87 | 1.23 | 2.19 | 14.07 | 1.95 | +| iq3xxs | 2.18 | 10.22 | 1.25 | 2.04 | 10.39 | 2.48 | +| iq4nl | 2.78 | 2.20 | 0.95 | 2.42 | 2.73 | 3.03 | +| k2 | 2.06 | 1.49 | 1.14 | 2.28 | 2.66 | 1.70 | +| iq2s | 1.33 | 6.70 | 1.21 | 1.18 | 6.90 | 1.19 | +| iq2xs | 1.43 | 6.50 | 1.31 | 1.56 | 8.08 | 1.65 | +| iq2xxs | 1.79 | 7.64 | 1.44 | 1.65 | 7.26 | 1.65 | +| mx4 | 2.60 | 4.57 | 1.00 | 2.52 | 4.80 | 2.60 | + +Reading: every one-thread row on both boxes is past the reference (zen4 >= 1.10, Intel >= 1.18); every grid row at 16 lanes is 1.19-2.48x. +Under 0.95 at 16 lanes: zen4 q8 0.89, iq4xs 0.92, q51 0.94 (the DRAM-bound rows; iq4nl exactly 0.95) and Intel k6 0.68 (the normal-mode stall gap). +Both minted profiles committed; the gemv seats took grid_vbmi for all five grid families on both boxes. + +### Intel v8 - THE GOAL TABLE (2026-09-01, c8i.4xlarge Xeon 6975P-C, x86-amx re-minted with the L3-proof engine-shape seat race, a3a1a08ab; ratio = reference / ours) + +| fmt | gemv 1T | tile 1T | gemv 16L | +|---|---|---|---| +| q8 | 1.16 | 3.76 | 0.96 | +| q8s16 | 1.19 | 3.78 | 1.02 | +| k4 | 1.48 | 3.31 | 1.53 | +| k5 | 1.61 | 6.71 | 1.51 | +| k6 | 1.18 | 3.78 | 1.06 | +| q40 | 2.05 | 3.64 | 2.78 | +| q51 | 1.94 | 6.35 | 1.87 | +| iq4xs | 2.10 | 4.11 | 2.35 | +| k3 | 1.60 | 3.24 | 0.95 | +| iq3s | 2.13 | 14.08 | 1.86 | +| iq3xxs | 1.99 | 10.32 | 2.18 | +| iq4nl | 2.37 | 2.68 | 3.30 | +| k2 | 1.61 | 2.13 | 1.44 | +| iq2s | 1.15 | 6.83 | 1.17 | +| iq2xs | 1.55 | 8.00 | 1.64 | +| iq2xxs | 1.59 | 7.27 | 1.73 | +| mx4 | 2.44 | 4.78 | 2.79 | + +Every row at or past 0.95: the goal holds on Intel. q8s16 is the like-for-like q8_0 row (f16 scales); the f32-scale q8 row is informational (byte-capped at 0.944 in theory, 0.96 here). + +### zen4 v8 - THE GOAL TABLE (2026-09-01, c7a.4xlarge EPYC 9R14, x86-vnni512 re-minted with the L3-proof engine-shape seat race, a3a1a08ab; ratio = reference / ours) + +| fmt | gemv 1T | tile 1T | gemv 16L | +|---|---|---|---| +| q8 | 1.99 | 3.01 | 0.90 (ladder pass read 0.68; re-measured 6677-6703 vs 6059) | +| q8s16 | 1.92 | 2.99 | 0.96 | +| k4 | 2.29 | 2.49 | 0.98 | +| k5 | 1.33 | 6.30 | 0.98 | +| k6 | 1.07 | 3.47 | 0.99 | +| q40 | 4.48 | 3.42 | 0.91-0.97 (ladder 0.89; re-measured 2944-3143 vs 2851) | +| q51 | 2.83 | 6.02 | 1.01 | +| iq4xs | 2.17 | 4.05 | 0.91 | +| k3 | 1.79 | 2.69 | 1.01 | +| iq3s | 2.24 | 11.91 | 1.30 | +| iq3xxs | 2.22 | 10.23 | 1.19 | +| iq4nl | 2.86 | 2.20 | 0.99 | +| k2 | 1.74 | 1.69 | 1.10 | +| iq2s | 1.33 | 6.71 | 1.28 | +| iq2xs | 1.44 | 6.52 | 1.44 | +| iq2xxs | 1.79 | 7.66 | 1.51 | +| mx4 | 2.60 | 4.60 | 1.02 | + +One thread every row >= 1.07, tiles clean, every grid row 1.19-1.51 at 16 lanes. Under 0.95 at 16 lanes: the accepted streaming +set (ruled, followup_general 67) - iq4xs 0.91-0.94, k5 0.93-0.98, iq4nl 0.94-0.99, q40 0.91-0.97 - and the f32-scale q8 row, +byte-capped at 0.944 (q8s16, the GGUF q8_0 path, reads 0.96). The grid families' tile crowns are the vbmi rows (a tile tie - +the lattice row's tile IS the 512 body - so the gemv companion carries the lattice without a separate seat). + +### M4 Pro v2 - arm-i8mm re-minted on the rebased branch (2026-09-01; SOLO one thread, TEAM 10 perf-core lanes; ratio = reference / ours) + +| fmt | gemv 1T | tile 1T | gemv 10L | +|---|---|---|---| +| q8 | 1.65 | 2.31 | 0.95 | +| q8s16 | 1.62 | 2.24 | 1.02 | +| k4 | 1.40 | 2.18 | 1.00 | +| k5 | 1.10 | 3.80 | 1.09 | +| k6 | 1.34 | 2.22 | 1.11 | +| q40 | 2.08 | 2.37 | 1.24 | +| q51 | 1.24 | 2.91 | 1.56 | +| iq4xs | 1.68 | 3.22 | 1.14 | +| k3 | 1.27 | 2.60 | 1.46 | +| iq3s | 1.04 | 13.51 | 1.16 | +| iq3xxs | 1.09 | 12.41 | 1.18 | +| iq4nl | 1.85 | 3.18 | 1.30 | +| k2 | 1.44 | 2.54 | 1.52 | +| iq2s | 1.24 | 8.17 | 1.13 | +| iq2xs | 0.90 | 6.79 | 0.94 | +| iq2xxs | 1.17 | 8.58 | 1.22 | +| mx4 | 1.35 | 2.59 | 1.51 | + +Every row at or past 0.95 except iq2xs one-thread (0.90; 0.95 at the engine shape) - NOT a regression: v1's 1.46 does not reproduce with v1's own sources on the same box (both ~2350 us), and the row form beats the panel on ARM for every grid at both shapes (iq2xs panel 3885 vs rows 2365 at 10 lanes). followup_general.md entry 68 carries the true standing and the two kernel-work directions; chunk grain is a null (8/16/4/1 all ~2350). + +## 3. Research memos (read before touching the kernels) + +- `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, + x86 and arm64, against our emitter arms. Headline: every one of our five gathers ends each + decoded dword with a 4-byte store into `te.iq3s_panel` and reloads 32 bytes spanning eight such + stores - a wide load over narrow stores, which x86 does not forward. llama.cpp composes grid + words into registers on both arches. The signs-on-activation half of followup 61 does not port + to our 8-rows-per-vector layout; llama.cpp's arm64 arms put signs on the weights for the same + reason. Ranked spellings: (1) `gather="reg"` - group-major compose, `insertelement` per row, + no panel; (2) `psign="mask"` - a vector sign mask instead of GPR sign math + `llvm.abs`; + (3) a `vpdpbssd` seat (AVX-VNNI-INT8 hosts, no generator change); (4) repack-baked parity sign + byte (free for IQ2_XXS/IQ3_XXS). Incidental waste: IQ2_XXS reads its aux word as four i8 loads + + shifts where IQ3_XXS uses one i32 load; IQ2_XS builds a u16 from two i8 loads. +- `kernel_parity_research_zen4.md` - why llama.cpp's five grid kernels beat ours on zen4 while losing on zen2. + Headline: the reference has NO AVX-512 path for them - a znver4 build runs the same AVX2 body as zen2 (256-bit + vpmaddubsw, never vpdpbusd) and its 1.4-1.9x zen2 -> zen4 gain is the core alone; ours gained 1.14-1.26x because + we are stall-bound (fewer instructions, more cycles): the panel round trip (1024 scattered 4-byte stores per + superblock-group re-read as 64-byte vectors) is the one structure we have and they do not. Ranked: the row form + for iq3xxs on vnni512 (measured; loses at 16 SMT lanes - needs bare metal), `gather="qpanel"` (u64 grid entries + stored whole, est 1.2-1.4x on the iq2 three), the parity-completed sign byte baked at repack (5-8%, free for + iq2xxs/iq3xxs), mask-register signs at width 512 (vptestmb + merge-masked vpsubb), the 256-bit gemv seat (the + harness's own crown now), and the VBMI symbol lattice - the grids are a 3-symbol (iq2) / 8-symbol (iq3) + alphabet, so an entry is 12-16 bits and the decode is vpermi2b plus a few widen/shuffle ops, est 2-3x, needs + avx512vbmi in TUNE_KNOWN_FEATURES. Do not seat hardware gathers (llama.cpp measured them losing on a 7950X) + nor vpdpbssd (zen4 has no AVX-VNNI-INT8). +- `kernel_parity_research_vk.md` - llama.cpp's cm2 `mul_mm` against our cm2 tile. Headline: the + same kernel design (tile geometry, workgroup, decode-in-load callback, one coopMatMulAdd per + k-step, zero barriers in the k loop, the same split-k heuristic) - the gap is not the tile. + Real differences: llama.cpp hoists Q4_K/Q5_K scales into shared memory as ready `(d,m)` pairs + (`shAscales`), ours re-extracts from a second SSBO with a variable shift per decoded element; + its scales sit in the quant block, ours in a separate plane. Ranked: (1) split the end-to-end + pp512 budget first - `GGML_VK_PERF_LOGGER=1` per-shape MUL_MAT rows against our `pfq_ts` roles + decides whether the GEMM is even the gap (367 barriers per window in the chain); (2) the scale + hoist (10-20% on k4/k5); (3) interleave the scale plane into the quant block; (4) pad N to the + tile width; then narrow-tail accumulator, unroll, vec4 decode. Corrections: followup_vulkan 34's + "same band" (it is 0.67-0.90x), "shared tier not decode" (Q8 rows sit at 1.00x+), and "tg holds + parity" (IQ2_XS 0.54x) are stale. glslc here lacks NV_cooperative_matrix_decode_vector, so + llama.cpp runs its scalar decode too - apples to apples today, and its vec4 twin is a widening + waiting to happen. + +## 4. Work queue (one item at a time; a row is done at >= 1.0x on rig 2's shape AND the vehicle) + +NEXT SESSION (Boris's rulings, 2026-09-01 evening): the gemv's own crown. The tuner races the TILE and the gemv +follows (`tune_companion`); on zen4 and Intel the gemv prefers `vpdpbusd_width256` where the tile crowns width +512 (iq2s 3-6%, k4 similar), and on Intel the stamped k6 gemv runs 1.5x slower than the same seat raced. The mint +races the gemv at decode shape between the TILE'S TOP TWO seats only (not the whole grid - "i don't see the point +of racing anything but top 2") and stamps the gemv separately; more mint time is fine ("performance matters +more"); `x86-amx` stays a class of its own, split from `x86-vnni512` ("for sure"). Then re-mint every class on +the emitter it ships with: x86-avx2 on zen2, x86-vnni512 and x86-amx rented (aws_bootstrap.sh, 35 min each), +arm-i8mm on the M4, arm-neon on the M1. After that: the zen4 grid kernels (the vnni512 panel form's compute), then +Vulkan gap 1. The CPU PR follows the Metal PR's merge (the make_pr chain on this branch). +LANDED (0251e8b2d): the kq grids gain `dot=vpdpbusd width=256 mr=16` (the same-layout alternative to the 512 tile +crown - the layout companion pins the plane mr, so only same-mr rows can differ); the harness picks the gemv among the +tile winner's same-mr rows, the two best by tile time, by the streamed-decode time past the margin over the tile +seat's own gemv (incumbent keeps a tie) and writes it under `q8_gemv_gen`; llvm_tune stamps a companion from +its own entry when it names an eligible perm of the grid. TEST 90 -> 103 rows on zen2 (the new rows decline there). +The arm classes and x86-avx2 have no same-mr alternative (their grids collapse to one eligible row), so only +x86-vnni512 and x86-amx need the re-mint: two rentals with `aws_bootstrap_mint.sh` (build, TEST, the 1B vehicle's +--tune, export, both ladders) - running. The export copies every kernels entry, so the gemv seats ride into the +class profiles; the profile adoption merges them as ordinary entries. +zen2 smoke of the new harness path (the 1B vehicle, --tune, 16 lanes): provenance ok / ok / avx2;f16c;fma;sse4.2, +49 kernels, every crown unchanged (maddubs256_mr8), no gemv seat written - correct for a class with no same-layout +rival. The x86-avx2 profile stands. + +CPU decode (gap 2) - the k-quant decode kernels were the tails the full ladder exposed (k3 0.80x, +k6 0.90x, k5 0.91x): +1. DONE k3 + k2: one i16 flush per sub-block (the per-16 chains are bounded at 4 x 1778 / 4 x 762) - + k2 decode 2061 -> 1286-1423 us (reference 1951-2121); and k3's decode re-loads its 24 shared + qs/hmask columns per sub-block (volatile loads, decode shape only) instead of CSE keeping them live + and spilling - k3 2828-3198 -> 2445 us (reference 2638-2773). The pin hurt the tiles (k3 776 -> + 1109 ms) so it is decode-only; the principled form is the memo's D1 group-major reorder. +2. OPEN k6 0.87-0.90x, k5 0.87-0.91x: the pin gained nothing on k6; the memo (research_cpu_kquant.md) + sees no clean lever for k5 (D4) and offers a sub-block unroll knob (D5) for both. + 2026-09-01, the k6 swing dissected (zen2, one thread, fresh process each, us): alone 4153 / 4225 / + 5468 / 4711; pinned to ONE core (start /affinity) 4040 / 5804 / 4220 / 4252 and 4138 / 4368 / 5632 / + 5200 - it follows the process, not the core. k4 (1999-2005), k5 (3590-3760), k3 (2648-2850) and the + reference q6_K (3664-3768) do not swing. Arena base page-aligned (`--base-align 4096`): 4862 / 5971 + / 6052 / 5916, at 65536: 6024 / 6364 / 5607 / 5567 - eight of eight slow, where the heap's 64-byte + draws run 4.0-4.3 two times in three. The sub-page phase sweep (`--base-offset` 0..3840 by 256, one + process each) reads 4383-6305 with no band (3840 -> 4383, 3584 -> 6231). Reading: the per-process + draw is the L1-set phase of k6's spill slots (the native stack, ASLR) against the planes; k6 has the + largest live set of the four (memo: 2 qh + 4 ql + 8 chains). The engine's image planes start on 16 KiB + boundaries (ARCHITECTURE_IMAGE.md 2.1a), i.e. the slow phase - the engine's k6 decode is ~0.61x, not + the ladder's 0.69-0.87. Spills counted in the cached JIT DLL (`objdump -d` over the `.map` range of + the `implementation` symbol): k6 72 ymm spill stores + 95 stack ymm reloads per superblock body (1224 + instructions, 64 maddubs) against k5 20 + 8 and k3 42 + 13. Pinning the qh column loads (volatile, the + k3 cure) cut the stores to 27 but left 90 reloads and made k6 uniformly SLOW - engine phase 5537-5895, + heap phase 5574-6078 (the 4.0-4.3 mode gone), tile 634 -> 669 ms - so it was reverted; the memo's + "the pin gained nothing on k6" stands, and the fast mode is not the spill count. NEXT (its own session, + with counters - AMD uProf on this box): what the 4.0 and 6.0 modes differ in, at `--base-align 4096` + with `--base-offset` as the phase knob; only then D2's hoist / D5's unroll. + 2026-09-01, Boris's ruling - the .dlim is hardware-specific (Vulkan and Metal bake their own; the + flavors never co-exist), so the CPU flavor's planes are the CPU kernels' to lay out. k6's qh half + was GGUF's packing copied verbatim (four SUB-BLOCKS per byte), which is why the kernel kept eight qh + columns live across four sub-blocks or reloaded them. LANDED: the qh plane re-packed per sub-block + (column 2blk + half holds that sub-block's four j sites at 2j) - same 64 bytes, two qh loads per + sub-block, uniform shift, nothing lives past its sub-block; six readers moved (repack, tile unpack, + gemv arm, row dequant, generic tail dot, layout inverse), IMAGE_VERSION 27, JIT 0x5b. TEST 90/90, + test_gguf_quant / test_repack / test_kquant green. zen2 k6 decode, us: engine phase 4444 / 4619 / + 4708 (was 4862-6364), heap phase 4749 / 4729 / 5427 (was 4040-5804); spills 72 + 95 -> 36 + 67; + TEAM=16 wall 715 / 759 (unchanged, balance-bound); tile 659 ms (unchanged). The engine's k6 goes + ~0.61x -> ~0.80x for free; the 4.0 ms mode is not reached - the remaining spills are the i16 + chains and broadcasts. + LANDED k3 (a387ed230): qs per sub-block at 2j (two loads) and hmask one column per sub-block at bit + s (one load) - three loads per sub-block, the volatile pin gone, same 96 bytes; IMAGE_VERSION 28, JIT + 0x5c; TEST 90/90 + the three module tests green. zen2 one thread, us: decode 2590 / 2925 / 3012 + heap phase, 2555 / 2825 / 3504 page-aligned, against 2648-2850 with the pin (reference 2701) - + equal at one thread with a wider draw; the TILE 733 -> 686 ms (the pin was decode-only, the layout + is not). Body 1027 instructions, 41 + 39 spill ops (was 1152, 42 + 13 with the pin). k5 is already + site-major (one qh byte per row per site) - its cost is the byte -> lane expansion (shuffle + two + masked tests per site); the k3 hmask shape fits it bit for bit: one column per sub-block, deposit + `(hb << (4 - j)) & 0x10` / `(hb >> j) & 0x10` - six ops per site against ten plus a shuffle. + DONE-KILLED k5 (2026-09-01): the transpose was correct (TEST 90/90, the three tests green) and + SLOWER - decode 4378 / 4414 / 4556 us heap phase, 4007 / 4664 / 5212 page-aligned, against + 3590-3760 before (reference 3307); tile 515 -> 559 ms. x86 has no byte-vector shift: `shl` / + `lshr` on <32 x i8> lower to word shifts plus masks, so the per-lane mask test (no shift) was the + cheaper deposit all along; k5's site-major byte and its shuffle stay. Reverted, nothing committed. + The transposes are done: k6 and k3 landed, k5 was already right. On the M1 (sdot lattice, same + IR): k6 1777 -> 1701, k3 1925 -> 1873 us, TEST 90/90. + LANDED the fused scale flush (x86 maddubs lattice, k6/k3): the per-16 sub-scale rides the chain's + widening - `pmaddwd(chain_i16, scale_i16_pair_splat)` straight into iacc, exact in i32 (|chain| <= + 32004 x |scale| <= 128) - instead of pmaddwd(ones) + i32 vpmulld into a0/a1 and a second add. 32 -> + 16 vpmulld, a0/a1 gone from the live set, k6 spills 36 + 67 -> 31 + 52. zen2 k6 decode at the + engine phase 3527 / 3636 / 3851 us against the reference's 3683 (was 4444-4708): PARITY; heap 3808 / + 4007. k3 engine phase 2488 / 2613 / 2893 (reference 2701, was 2555-3504). Tiles unchanged within + noise (k6 588-702 across runs, k3 660). TEST 90/90. The M1 is untouched (sdot has no i16 chain). + Extended to k4/k5 (one 6-bit scale per sub-block, zero-extended): TEST 90/90; k5 decode at the engine + phase 3399 / 3474 / 3511 (reference 3307, 0.94-0.97x; ~3600 before), k4 2030-2151 (reference 2435), + tiles unchanged (k5 505, k4 482 ms). k5's last ~4% is the high-bit deposit itself (shuffle + two + masked tests per site) - the per-sub-block layout lost to byte-shift emulation, the memo's D4 stands: + k5 closes at 0.95x on zen2, 1.06x on the M1. + +The 16-lane row and what it measures (2026-09-01, `--team`, the engine's own splitter: 64 self-served +chunks of 64 rows): k4 wall against the reference at the same thread count - 8 lanes 648 vs 675 us +(1.04x), 16 lanes 469 vs 341 (0.73x), 32 lanes 342 vs 112 (0.33x); k6 at 8 lanes 1165 vs 1041 +(0.89x). The reference at 32 threads moves 33 MB in 112 us = 295 GB/s, three times this box's DRAM: +`test-backend-ops perf` repeats one small op and its row -> thread mapping is stable, so the weights +stay in the 3990X's 256 MB of L3 (16 MB per CCX); our self-served chunks move rows between CCX slices +every call and pay DRAM. At 8 lanes, where neither side gets the L3, we are at parity. A model's +weights (4-30 GB) never stay in L3 between tokens, so the fair many-lane comparison is a working set +above L3 - d=32768 (k4 264 MB) - measured next. If parity holds there, the many-lane gap seen at +m=4096 is the microbench's artifact, not the dispatch. +MEASURED (the reference given the m=32768 perf row - HOW_TO_GET_SIDECAR; the ladder's `BIG=1`): +k4 16 lanes 4560 vs 4549 us (1.00x), 32 lanes 4117 vs 5147 (1.25x); k6 16 lanes 6534 vs 6474 +(0.99x) - both sides at the box's ~58-64 GB/s DRAM rate. PARITY at the engine's shape; the +"dispatch grain" question is closed - nothing to fix in the splitter. The m=4096 many-lane rows +stay in the ladder as a kernel-in-L3 view only; the BIG row is the one that speaks for decode. +3. Noise: the one-thread bench and the reference both drift ~5-8% run to run; interleaved rounds hold + ours steady, the reference is re-run per table. iq3xxs/iq2s/iq2xxs tie at 1.00; k2 now clear. + +CPU decode, the grid formats (done): +1. DONE-KILLED `gather="reg"` (measured 1.85x slower - see the ledger). +2. DONE all five: `sign="vec"` - the sign-byte column negates whole vectors; iq3s/iq2s load it + off the plane, iq3xxs/iq2xs/iq2xxs synthesize it from the 7-bit codes (parity = the 8th bit), + so no plane layout change (the repack-baked byte was killed by the plane map: grp bytes are the + disk bytes, ~30 CPU/Vulkan/Metal sites read them). +3. DONE: the iq2 formats' u64 grid pair as one 8-byte load, half split in registers. +4. DONE: the vector sign column IS the gemv path - the `sign` knob and its 20 seats are gone. The + tuner races the TILE (a gemv-only spelling ties there; `--tune-only iq3sq8_tile_gen` crowned the + old seat), and the M1 (sdot lattice, one thread) showed no losses: iq3s 7383 -> 6105 us, iq2s + 7161 -> 5012, iq2xxs 7602 -> 6293, iq2xs 7878 -> 7081, iq3xxs 7516 -> 7452. Shipped profiles + stay valid (same seat names); LLVM_JIT_CODEGEN_VERSION 0x5a re-keys the JIT caches. +5. DONE where it won: the column dword read with in-register byte extraction - iq3xxs 7862 -> + 6602 us and iq2xs 6501 -> 4702; it LOST on iq3s (7537 -> 8671), iq2s (5202 -> 6187) and iq2xxs + (5121 -> 5574), which keep byte loads. The u16 single load for iq2xs measured slower (6991 vs + 6306) and was dropped. +6. Residue at one thread: iq3xxs 0.99x, iq2s 0.98-1.01x, iq2xxs 0.93-1.00x (the noise band of the + 1-thread bench is ~7%; a longer run or the 16-thread engine shape decides the last 2%). 7. the + 16-thread stamp on the vehicles (needs `--for-debug-purposes` on lcpp_bench, or the released + exe). 8. retro audit of IQ4_XS/Q3_K per followup 60. + +CPU decode, grid formats, BOTH ISAs (M1 0.51-0.93x, zen4 0.77-0.88x): the scalar per-dword gather is +the bound; the fix is the vectorized u64-lane compose (2 rows x 8 weights per NEON vector, 4 per 256-bit, +8 per 512-bit) with one pairwise add recovering the accumulator layout - one emitter change under the +gather branch, no plane change, retires the panel and the parity chain. k6 0.90-0.91 on both x86 boxes +stays open. +1. DONE ARM (2026-09-01, two commits): the row-pair decode under the sdot lattice + the ksigns +-1 + table rows + the whole-u64 grid load - M1 iq2xs 1.24x, iq2xxs 1.20x, iq3xxs 1.36x, iq2s 1.51x, + iq3s 1.15x (section 2, the LANDED notes). The x86 emission is byte-identical (gated DOT_SDOT). +2. DONE-KILLED x86 (2026-09-01): the row-QUAD form measured on zen2, us, panel -> rows: iq2xs 4646 -> + 6785, iq2s 5056 -> 6826, iq3s 7612 -> 11576, iq3xxs 6589 -> 7995, iq2xxs 5144 -> 7142 (TEST 90 ok, the + form is correct). llama.cpp's own AVX2 grid path IS this insert form and runs iq2xs at 5320 there - so + the insert shape loses to the panel on zen2 AND ours is slower than theirs in the same shape (the sign + table's four scalar loads + inserts where theirs shuffles; the per-row index math). It cannot close + zen4's 0.77-0.88x by being the reference's shape; the emitter keeps it for the sdot lattice only + (`grid_rows_path`). zen4 needs a profile first (which port saturates) - item 3. + The plan that was: four rows x 8 weights per 256-bit vector + (four u64 grid loads, vmovq/vpinsrq/vinserti128), the activation's 8 bytes broadcast per lane + (vpbroadcastq) and SIGNED per row by `vpsignb` against the same +-1 table rows - maddubs needs + its unsigned operand, so on x86 the signs go on the activation copy, which the 4-rows-per-vector + layout allows where the 8-rows-per-vector panel layout did not. Then vpmaddubsw + the existing + fold. The zen2 `gather="reg"` that measured 1.85x slower was insertelement PER DWORD with GPR + sign math; this is four qword inserts and no sign arithmetic. Measure on zen2 first (all five + at 1.00-1.36x there - it must not lose), then zen4 (rent) where the gap lives. +3. NEXT x86 zen4 grid 0.77-0.88x: rent the c7a again with `perf stat` (uops per port, store-forwarding + stalls) on iq2xs/iq3xxs decode against the reference exe's same shape - one proven fact before any form. + 2026-09-01, the fact the two boxes give for free: zen2 -> zen4 sped every kernel up 1.5-3x (k4 2337 -> + 833 us, k6 3460 -> 2866) EXCEPT the grid decodes (iq2xs 4645 -> 4562, iq2s 5058 -> 4918, iq3s 7603 -> + 7571) - a wider, faster core buys them nothing, so they are bound by a latency chain (index -> grid + load -> panel store -> reload), not by vector throughput. That is the case for the register compose + on zen4 even though zen2 killed it: `DASLLAMA_GRID_ROWS_X86=1` re-arms the x86 row-group arms as a + lab knob (env, read at emit time - clear `.jitted_scripts` between A/B runs, the JIT cache does not + key on it). A/B on zen4 next; if it wins there and loses on zen2 the gate becomes the tune class. + MEASURED on zen4 (one thread, engine phase; panel -> row form): iq2s 4918 -> 3559-3656 (reference 3757: + PARITY), iq3s 7577 -> 6267-6419 (6105; but -10% at 16 lanes), iq2xxs 4860 -> 4302-4485 (3922); iq2xs + 4548 -> 8072 and iq3xxs 5973 -> 10744 LOSE 1.8x with either read shape (dword column or bytes). The + sign path splits the ISAs too: the per-row +-1 table (the M1's win) costs x86 four scalar loads and + three inserts per group, the vector-synthesized sign column plus one mask is cheaper there - with it + zen2's row form reads iq2xxs 4696 (panel 4998) while iq2s/iq3s/iq2xs stay panel there. LANDED + (c6e99f81a, f68534c06): `grid_rows_path` = the x86-vnni512 class takes the row form for iq2s and + iq2xxs, every other x86 case the panel; the knob forces either. zen4 default check: iq2s 3595-3714 + (1.01-1.05x), iq2xxs 4383-4497 (0.87-0.89), iq2xs 4568-4577 (panel). OPEN: iq2s at 16 lanes on the + c7a (8 cores x 2 SMT) reads a 5.9 ms wall with one slot at 8.8 ms, both forms - iq2s's 8 KB grid table + plus panel plus x, twice per core, against a 32 KB L1 is the suspect (8-lane run next); the zen4 grid + residue at the engine shape is iq3s 0.78, iq2xs 0.86, iq3xxs 0.90 - the panel form's compute, kernel + work on the vnni512 lattice (a session of its own). + zen4 at 8 lanes (one per core, d=32768; reference at 8 threads): iq2s 6398 (4236: 0.66), iq2xxs 2852 + (4206: 1.47x), iq2xs 4582 (3975: 0.87), iq3s 7668 (6410: 0.84), k6 5013 (4878: 0.97, 77 GB/s = the + box's DRAM). iq2s scales at 56% of its one-thread rate (147 MB in 6.4 ms = 23 GB/s, not bandwidth) + where iq2xxs scales at 96% - the 8 KB grid table's L1 pressure is the suspect. + RESOLVED (the order test): the 8-9 ms slot belongs to whichever format is FIRST in a bench process - iq2s + first 4887 (slot 786-8747), iq2xxs first 1924 / 4063 (342-7920); the same format second is flat (iq2s + 3257 at 2861-3377). The team arm now burns in (six warmup dispatches per row). With iq2s second, its row + form at 16 lanes reads 3257 against the panel's 2525 (the ladder's row; reference 2159): the one-thread + win does not survive the engine shape, so iq2s left the vnni512 gate and iq2xxs alone stays (1828-1924 + against the panel's 2456). zen4's grid residue at the engine shape stands: iq2s 0.85, iq3s 0.78, iq2xs + 0.86, iq3xxs 0.90 - the vnni512 panel form's compute. + The frame realignment (every generated kernel's entry gets an alloca [64 x i8] align 64, so spill slots + never straddle a cache line whatever RSP the caller arrives with): zen2 k6 heap phase 3344 / 3437 / 3737 / + 4389 (was 4.0-5.4 with two of three slow), engine phase 3423 / 3593 (reference 3683: 1.03-1.08x). The draw + narrowed, one outlier remains - the stack phase was part of it, not all of it. + Verified after the burn-in (8693a5b47), zen4 16 lanes with iq2s FIRST: iq2s 2535 (the ladder's 2525; reference + 2159 -> 0.85), iq2xxs 1696 (row form; 2134 -> 1.26x), iq2xs 2309 (2000 -> 0.87). Gate final: x86-vnni512 -> + iq2xxs; x86-amx -> iq2xxs, iq3xxs; everything else the panel; the knob forces either. + Round four, the row form with the COLUMN signs on zen4 (one thread, engine phase; reference in + parentheses): iq2s 3541-3578 (3757: 1.05x), iq2xxs 2737-2771 (3922: 1.42x, from 0.89 with the table), + iq3xxs 4234-4322 (5243: 1.22x, from 0.56 with the table; the panel 0.88), iq3s 6238-6309 (6105: 0.97), + iq2xs 5381-5398 (3624: 0.67 - the panel's 0.80 stays). At 16 lanes on this 8-core x 2 SMT VM: iq2xxs + 1670 (panel 2456), iq3xxs 3415 (panel 3123 - loses), iq3s 4590 (panel 4145 - loses), iq2xs 2742 (panel + 2320), iq2s the 8.7 ms-slot anomaly. The one-thread wins that lose at 16 lanes point at SMT sharing + (two lanes' instruction streams per core); the 8-lane run decides what the class gate should say for + iq3xxs and iq3s. Tiles with the 64-aligned panel and scratch on zen4: unchanged (k5 178623, k6 225455, + iq3s 258990, iq2s 283977, iq2xs 284079 us). + +CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x. The memo +(`kernel_parity_research_arm.md`): the sdot count is at parity (2 per 32 weights per row, both sides); +the gap is the surrounding work - the parity chain that rebuilds the 8th sign bit for the three ksigns +formats (+3.25 cycles per row per 32 weights: exactly iq2xs/iq2xxs/iq3xxs), the u32 grids' doubled +loads, and the alloca panel, which in decode (tokCount 1) buys back nothing; a port model reproduces +all five M1 numbers within 10%. THE FIX: a 2-rows x 8-weights-per-vector decode form under the sdot +lattice - non-indexed sdot with the activation as `ld1r {v.2d}`, one `addp` per row pair recovers the +`<4 x i32>` accumulator the fold already takes; zero x86 exposure (inside the gather branch, gated on +DOT_SDOT), and it makes llama.cpp's +-1 sign table (`keven_signs`) reachable, retiring the parity chain. +Model: iq2xs ~2570 us = 1.27x. vqtbl gathers are impossible (1-2 KB grids vs tbl's 64 bytes); our +masked negate already beats llama.cpp's own mask expansion. Fallback if deferred: plane-carried sign +bytes (size-neutral for iq2xxs/iq3xxs, +6% for iq2xs; touches the plane, both ISAs) -> 0.72-0.98x only. +Also from the memo: the column-read-vs-byte-read decision was made on x86 and should be a perm seat. +LANDED 2026-09-01 (the row-pair decode under the sdot lattice, gated DOT_SDOT + decode shape + width +128; x86 untouched): M1 one thread, us, before -> after (reference): iq2s 4932 -> 3256 (4917, 1.51x), +iq3s 6089 -> 4937 (5682, 1.15x), iq2xs 6375 -> 4244 (3274, 0.77x), iq2xxs 6063 -> 4085 (3437, 0.84x), +iq3xxs 6735 -> 5235 (4867, 0.93x); TEST 65/65 on the M1. +LANDED 2026-09-01, step two - signs and grid words: the ksigns formats take a +-1 table row per 7-bit +code (keven_signs form, [128 x i64]) and the iq2 formats load the u64 grid entry whole. M1 us, after +(reference): iq2xs 2636 (3274, 1.24x), iq2xxs 2873 (3437, 1.20x), iq3xxs 3576 (4867, 1.36x), iq2s 3260 +(4917, 1.51x), iq3s 4929 (5682, 1.15x); TEST 90 ok. All five grid decodes ahead on the M1. LESSON: the +table for the plane-byte formats (iq3s/iq2s) REGRESSED them (iq2s 3256 -> 4167, iq3s 4937 -> 5608) - +four scalar loads per pair replaced one 16-byte column load plus four NEON ops, and the M1 sustains 3 +loads against 4 NEON ops per cycle; those two keep the column mask. Count loads AND NEON ops per pair, +the bound is whichever is fuller. + +Bench on AMX boxes: the tune-mode q8 tile SIGILL (above) - read how gen_tune_probe reaches the grant +(q8q8_family_live_variants + the amx cfg companion?) and do the same; k6 bimodality on Granite Rapids. + +Vulkan pp (gap 1): +1. Budget split (measurement, no kernel edit) - is the GEMM the 30%? 2. `shAscales`-style scale + hoist; 3. scale plane interleave; 4. N padding; 5+. the rest of the memo's list. + +Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. + +## 5. Ledger + +- 2026-09-01: CPU item 1 measured and killed - `gather="reg"` (compose the weight vector with + insertelement, no panel) ran iq3s decode at 90.6 ns/sb against the panel's 49.1; the emitted IR + had 1025 insertelement chains and a 9904-instruction straight-line body against the panel loop's + 1893 (every load unique, so CSE was not the problem). The panel round trip is not the bottleneck. +- 2026-09-01: CPU item 2 landed as `sign="vec"` on iq3s and iq2s, the two formats whose sign bytes + sit in the plane at row stride 4: the panel holds raw grid words, one 32-byte column load per + block, a constant shuffle + and + cmpeq builds the byte mask, xor + sub negates the vector. + One thread, m=4096 k=14336: iq3s 11581 -> 7732 us (1.34x of the reference's 10340), iq2s 12413 + -> 7076 us (0.72x of 5074, was 0.41x). Every variant bit-exact in gen_tune_probe TEST mode. The + five gather emitters collapsed into one gather over per-format decode functions (-110 lines). +- 2026-09-01: research_cpu_kquant.md (llama.cpp q3_K/q5_K/q6_K vs ours): op counts are at parity; what + tracks the losses is the high-bit plane's column vectors staying live across the superblock after + CSE (k3 24 vectors, k6 16) against 16 ymm - the plane cannot be register-resident in our + 8-rows-per-vector layout the way it is in llama.cpp's row-major loop. Also: llama.cpp's q4_K has an + AVX2 repack GEMV that test-backend-ops perf does not run, so the k4 1.22x is against its fallback. +- 2026-09-01: the lab completes - kq_kernel_bench carries all 16 formats (q8 + mx4 on the q8q8 + grid, q51 on its per-32 planes; the tile arm hands k5/k6 and the grid formats their byte-expanded + panel per kq_reads_packed_planes), rows interleaved round-robin with best + median; + harness/kernel_ladder.sh joins both sides into the per-box table (above). +- 2026-09-01: the 16-thread vehicle stamps (above) - every grid-format vehicle at or past the + reference on tg except the mixed i1-IQ3_XXS at 0.97x with a +-6% error bar. +- 2026-09-01: the column dword read lands for iq3xxs and iq2xs only (measured per format, see the + queue); zen2 one thread now: iq3s 7673, iq3xxs 6641, iq2s 5039-5174, iq2xs 4831, iq2xxs + 5281-5487 us; 65 variants ok in TEST mode. +- 2026-09-01: the sign knob collapsed into the one gemv path (-268/+58 lines in the emitter, the + smask/ksigns globals gone); the plain seats now measure iq3s 7537, iq3xxs 7862, iq2s 5202, iq2xs + 6501, iq2xxs 5121 us; 65 variants ok in TEST mode; test_kquant 265 passed. +- 2026-09-01: sign=vec for iq3xxs/iq2xs/iq2xxs via the parity-synthesized column (no layout + change) and the u64 grid pair load for the three iq2 formats: iq2s 5152 us (0.98x), iq2xxs 5209 + (0.98x), iq2xs 6297 (0.86x), iq3xxs 7893 (0.83x), iq3s 7562 (1.37x); 85 variants ok in TEST mode. +- 2026-09-01: re-mint round on the shipping emitter (commit 11f69493e). zen4 c7a (x86-vnni512, 50 + entries, TEST 103/103): the k3 tile crown moved 512_mr16 -> maddubs_width256_mr8 (tile race 51347 + vs 52840 us; the transposed k3 planes made the 256 maddubs seat the tile winner) and the k3 gemv + took its own same-layout seat vpdpbusd_width256_mr8 (374 vs the tile seat's 401 us) - the first + entry the gemv's own crown wrote; q51 tile 512_mr16 -> vpdpbusd_width256_mr8; axpy_q8kv plain -> + vec8_u2, rope_scaled_neox_tab vec8_u2 -> plain. Intel c8i (x86-amx, 49 entries, TEST 103/103): + every crown holds (k3 41768 vs 57342, k6 33760 vs 53832 for the 512 tile), axpy_f16 vec16 -> + vec16_u2, no gemv seat - so the Intel k6 normal-mode 5.2 ms gap stays open. +- 2026-09-01: the new dot_vpdpbusd_width256_mr16 row was a dud on both boxes - its tile time equalled + the reference row's (zen4 3168491 vs 3167365 us): perm_declines budgeted 16 ymm registers at width + 256, so a two-vectors-per-group row (nrsplit 4: 2*4*2+6 = 22) declined into the reference body. A + row that requires AVX-512 runs where EVEX encodes ymm16-31; the budget is 32 there (461edda72, + LLVM_JIT_CODEGEN_VERSION 0x66). The row is un-raced until the next mint. +- 2026-09-01: the qpanel spelling (DASLLAMA_GRID_QPANEL=1: eight qword stores into the alloca panel + + one width load instead of the insert chain, commit eec5ce80c) measured on zen2 with the row form + forced, one thread at the engine phase, qpanel 0 -> 1: iq2xs 8797 -> 8776, iq2s 6733 -> 7435, iq3s + 11886 -> 11647, iq3xxs 7212 -> 7262, iq2xxs 4624 -> 5841 us. A loss or a wash on zen2 (where the row + form itself loses to the panel form: iq2xs 4565 panel); the zen4 A/B is the one that matters + (iq2xxs's crown there IS the row form) - queued behind the box's ladder. +- 2026-09-01: qpanel killed - the zen4 and Intel A/Bs (row form forced, one thread, d=32768, qpanel 0 -> 1): + zen4 iq2xs 33680 -> 40895, iq2s 47037 -> 44739, iq3s 68550 -> 70264, iq3xxs 43468 -> 46495, iq2xxs 22687 -> + 23111 us; Intel iq2xs 44067 -> 43923, iq2s 44660 -> 56802, iq3s 84893 -> 86998, iq3xxs 39726 -> 39662, iq2xxs + 22041 -> 21450. A loss or a wash on all three boxes; the helper and the knob are gone from the emitter. + Same run, a find: on zen4 the row form beats the panel form for iq2xs at 16 lanes (2194 vs the ladder's panel + 2559 us - 0.91x of the reference instead of 0.78x) where the class gate picks the panel; the forms probe + (zen4_forms.sh, both forms x five grids x two shapes) decides the gate. +- 2026-09-01: the width256_mr16 row raced alone on both boxes (post probes, d=32768, tune mode; full tables + tmp zen4_post.log / intel_post.log). At 16 lanes every seat of every format sits within ~5% on both boxes - + the minted crowns stand at the engine shape. One-thread streamed decode: on Intel 256_mr16 beats the 512 + gemv for k3 (12214 vs 16462), k5 (18972 vs 21184) and k6 (27321 vs 31723) while its tile loses ~14%, so + the gemv's own seat is expected to take it at the next Intel mint; on zen4 it loses everywhere (k3 16193 + vs 7280). Intel iq2s: ALL seats ~0.90-0.94 of the reference at 16 lanes - the seat is not iq2s's lever. +- 2026-09-01: Intel k6 one thread, 40 rounds --each, normal mode: 3425, 1989, then steady 1637-1642 us - + the slow reads were WARM-UP, steady state is 1.16x of the reference (1882); the ladder's 5-round + best 1831 / median 2925 sampled the ramp. The 16-lane arm decides whether the 0.83 row is the same artifact. +- 2026-09-01: the VBMI lattice is CORRECT and WINS (Intel c8i, first races): TEST bit-exact on all five + grid families (grid_vbmi maxdiff <= 2e-6); iq2xxs one thread 19334 vs the 512 row's 21823 us, 16 lanes + 2249 vs 2599 (the reference 4254 - 1.89x); iq3xxs one thread 30438 vs ~39700 (the panel form's post-probe + read). The vbmi row's tile is the 512 body, so the tile race ties and the gemv's own seat crowns the + lattice where it wins - a plain re-mint adopts it, no framework change. +- 2026-09-01: Intel k6 16-lane mode gap CONFIRMED mode-specific, not warm-up: same pinned seat, steady state + ~4900-5300 us normal vs ~3236-3400 tune over 20 rounds; one thread differs 1% (1629 vs 1616). Arena phase + is an 8% one-thread effect (off 64 best 1764, off 128-256 ~1920), not the 1.5x. intel_k6perf.sh counts the + steady loop (perf stat + record by DSO) in both modes - kernel pages vs dispatcher decides the next step. +- 2026-09-01: the lattice's full Intel race (tune mode, d=32768; vbmi / old 512 seat / the reference at + 16 threads): 16 lanes iq2xxs 2249 / 2599 / 4255, iq3xxs 3376 / 4536 / 6704, iq3s 3726 / 7212 / 7413, + iq2xs 2815 / 3951 / 4610, iq2s 3316 / 5706 / 3848; one thread iq2xxs 19334 / 21823, iq3xxs 30438 / 39773, + iq3s 36485 / 72793, iq2xs 23533 / 32658, iq2s 28428 / 36371 us. Every Intel grid row clears 0.95 with the + lattice - iq2s (the 0.90 row) reads 1.16x. The zen4 half runs next; then the re-mints adopt the seats. +- 2026-09-01: the lattice's zen4 race (tune mode, d=32768; vbmi / old 512 seat / the reference): one thread + iq2xxs 17571 / 22402 / 32946, iq3xxs 27384 / 48023 / 43551, iq3s 26120 / 61810 / 49870, iq2xs 20518 / + 36590 / 30978, iq2s 26909 / 39628 / 32855; 16 lanes best iq2xxs 1551 / 1666 / 2106, iq3xxs 2520 / 3108 / + 2810, iq3s 2778 / 3882 / 3231, iq2xs 1618 / 2334 / 2006, iq2s 1935 / 2516 / 2113 us. TEST bit-exact. + Every zen4 grid row clears 0.95 with the lattice - the four 0.76-0.90 one-thread rows read 1.2-1.9x. +- 2026-09-01: the Intel mode gap narrowed to the memory system: warm cache, both modes 96%+ of cycles in + the kernel DLL, same clock, same THP rollup, and the instruction rate ratio equals the wall ratio exactly + (identical instructions per gemv) - the normal body just stalls 1.54x more, 16 lanes only. Tune streams + 385 MB at 127 GB/s (partial cache residency), normal at 82.8. Round 5 sweeps d (8k/32k/64k), adds q8, and + reads the arena VMA's own AnonHugePages plus LLC counters per mode. +- 2026-09-01: cold-start SIGSEGV at 0xc0 right after "Library linked - ok" (normal mode, 16-lane team k6 + d=32768, Intel; reproduced 3x cold, never warm, earlier bootstrap ladders passed) - a post-link rebind + bug to chase before the PR. +- 2026-09-01: the cold-start SIGSEGV bisected. It needs only the PRESENCE of the five grid="vbmi" + tune_perm rows in dasllama_math_gen.das - not the emitter body, globals, or intrinsic decls (zen4 B1/B2/B3 + disable each in turn, all still crash), and not the startup race (zen4 T1: fingerprint patched so nothing + races, rows present -> crash; T2: rows removed, k6's own row made to race -> clean). It fires only cold + + normal mode + --team; warm run 2 and every tune-mode race are clean, and the crashing format is k6, not a + grid format - so it is a whole-module codegen/teardown effect of the extra generated variants (backtrace + LLVMContextImpl::~LLVMContextImpl, fault at a small offset), not the lattice IR. Correctness is unaffected + (TEST bit-exact, all races produced numbers). A focused C++ session owns the fix; the parity numbers come + from warm/tune runs meanwhile. +- 2026-09-01: the cold-start crash FOUND and FIXED (17e664e7f). Cuts on zen4: L1 (permi2b calls, no table + loads) clean vs L2 (table loads, no permi2b) crash pinned the lattice's 64-byte table-part loads; the core + showed glibc's malloc_consolidate under ~LLVMContextImpl - heap corruption at compile time. Cause: the + llvm_boost LLVMBuildGEP2 wrapper's `inbounds = true` default called LLVMSetIsInBounds on the result, and a + GEP over a global with a constant index folds into a ConstantExpr - the cast to GetElementPtrInst wrote a + flag into the constant's memory. Every emitter GEP before the lattice had a variable index or a parameter + base, so nothing ever folded. The wrapper now builds inbounds GEPs through LLVMBuildInBoundsGEP2. Cold + SOLO and cold 16-lane team both pass on zen4. Dead ends on the way: startup race path (T1/T2), frame size + (five by-value grid tables), constant folding of the loads, split partitions, lane count. +- 2026-09-01: Intel re-mint on the lattice emitter (x86-amx, 60 entries, provenance ok/ok, fingerprint carries + avx512vbmi): the gemv seats took grid_vbmi for all five grid families and 256_mr16 for iq4xs. The goal + ladders after it: one thread every row >= 1.18 (k6 1.19, iq2s 1.18, iq3s 2.19, iq3xxs 2.04, iq2xs 1.56, + iq2xxs 1.65); 16 lanes every row >= 0.95 (q8 exactly 0.95, iq3s 1.95, iq3xxs 2.48, iq2s 1.19, iq2xs 1.65, + iq2xxs 1.65) EXCEPT k6 0.68 (5346 vs 3635 us) - the normal-mode stall gap, Intel's last open row. +- 2026-09-01: cold-start cost is LLVM, not the das compile: the mint's JIT line reads optimize 60 s + emit+link + 72 s for the whole dasLLAMA module - the lattice bodies are fully unrolled over the 8 blocks (a called block + loop would cut it); -module-cache trims the das side only. +- 2026-09-01: the Intel k6 16-lane mode gap VANISHED after the rebase onto master (d76d86fe0 + binary + rebuild): normal 3434 us vs tune 3287 / 3119 (was 5323 vs 3698 for the identical 512 body); answers + bit-identical across modes (the bench's new '# ysum' line). Two facts changed at once on the pre-rebase + tree: the stamped gemv companion carried the TILE's 512_mr16 body although the profile named the 256_mr16 + seat (the disassembly: outlined 491-insn EVEX body called from the das thunk), and the same 512 body ran + 1.44x slower in normal mode than as a tune-mode clone - both gone now (normal stamps the VEX 256 seat). + Not isolated to a master commit; if the gap ever returns, bisect llvm_tune.das between + bbatkin/kernel-parity-pre-rebase and the rebased tip. Ruled out along the way: alignment/arena phase, + THP, L3 residency, dispatch grain, thread placement, per-call interop cost, outlined-vs-inlined body. +- 2026-09-01: zen4 16-lane rows are NOT a mode gap (tune == normal: q8 6715 / 6733, iq4xs 3077 / 3008). Grain + matters for balance: at --chunks-per-lane 16 iq4xs 2978 (0.96 of 2856) and q51 4132 (0.99 of 4093) with + balanced lanes; q8 stays ~6700 at every grain (0.90) with every lane busy the whole wall - 70 GB/s against + the reference's 82 on q8 while our k4 streams at 91 on the same box: memory-level parallelism, not bandwidth. +- 2026-09-01: x86 hybrid core detection (Apple-only today) is LEDGERED as followup_general.md entry 66, not an arc + follow-up; the goal boxes are homogeneous (c7a EPYC 9R14, c8i Xeon 6975P-C - no cpu_atom PMU). +- 2026-09-01: the q8 ladder row was a byte handicap: the bench's q8 plane carries f32 group scales (9 bits/weight) + against llama.cpp's q8_0 f16 d (8.5) - at equal streaming rate it can never pass 0.944. The engine runs GGUF + q8_0 tensors on the wscale_f16 twins, so the bench got a q8s16 arm (2ad5b9980): zen4 16 lanes 6277 vs the + reference 5985 = 0.95 (f32 row 6673 = 0.90), both at ~79.5 GB/s. The pf=2048 software prefetch was a null + (6712 / 6722 / 6712 for pf / crown / 256_mr8). +- 2026-09-01: zen4 lane sweep at d=32768, 8 vs 16 lanes: iq4xs 3004 / 3116, iq4nl 2946 / 3049, q8s16 6120 / 6339, + k4 2891 / 3169 (streaming formats gain 3.5-9% at one lane per core - issue width at two SMT lanes, not DRAM), + k5 3997 / 3949 (flat), iq2xs 2714 / 1412 (the lattice grids halve without the SMT lanes). The Mac decode policy + transfers to SMT x86 PER FORMAT. RULED (Boris, 2026-09-01): ledgered as followup_general.md entry 67, not built + this arc - zen4's iq4xs 0.92 / iq4nl 0.94 / k5 0.93-0.97 are accepted; pinned affinity is normally faster and + this may be the one case it is not. +- 2026-09-01: Intel v6 (rebased, seat race at the engine shape but n=2048): every row >= 0.97 except k3 0.89 - + the tiled fixture's row length flipped the k3 seat to 256_mr16 again; the seat fixture is now built at n=14336 + (29dc0b35f). Intel v7 and zen4 v7 re-mints running. +- 2026-09-01: the board rig's parity pregate "flake" was a spawn defect, not the engine: `popen_argv` + builds a CreateProcess command line with a NULL application name, and Windows refuses a RELATIVE + argv[0] spelled with forward slashes (`modules/.../dasllama-bench.exe` -> spawn fails, rc -1, no + child) while `./x.exe`, `../x.exe`, absolute and backslash spellings all resolve. The rig spawned + exactly that spelling; the same child run by hand passed 40/40. Fixed at the source - the spawn + respells argv[0] natively on Windows (every `popen_argv` / `spawn_argv` / `popen_argv_pipe` caller), + `tests/fio/popen_argv.das` spawns the offending shape, `run_and_stream` now logs a failed spawn + instead of "pregate output above" with nothing above, and the caller-side backslash workaround + left `daslib/fio.das` and the language skill. +- 2026-09-01: the zen2 board row for gemma-4-12B Q4_K_M re-minted on the fixed rig (`records/zen2.json`, + 16 threads, reference unpinned as the 08-04 sweep ran it): das pp512 66.4 +-1.7 / tg128 6.74 +-0.05 + against the clean-cpu reference 56.4 / 6.45 = 1.18x / 1.04x; the 08-04 row read 70.2 / 7.30 against + 59.1 / 6.80 = 1.19x / 1.07x. Both engines sit ~5% under August on this box (a different reference + build too, 6c84c7d5d against ebd048f); the ratio held. Pinning the reference made no difference + today (56.8 pinned, 56.4 bare). The das side ran the adopted x86-avx2 class profile (sidecar sha + 3fe069fc370e), the 08-04 row a paranoid box mint. NOT STORED: the records gate (`performance/REVIEW.das`) + rejects a run whose `hardware.remote_desktop` reads `parsec`, and the stamp fires on the parsecd + daemon's presence - it runs as a service on this box - so the row backs out of the branch and the + cell is re-minted once Parsec is stopped for the ten-minute rig window (Boris's call; the daemon is + his remote access). Instrument: `gen_bench_records.das -- -o gemma-4-12B --workload llm + --ref-no-affinity`, both walls out-of-process, direction-grade against the 08-04 row. +- 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base + above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. diff --git a/plans/kernel_parity_research_arm.md b/plans/kernel_parity_research_arm.md new file mode 100644 index 0000000000..0729cfa936 --- /dev/null +++ b/plans/kernel_parity_research_arm.md @@ -0,0 +1,492 @@ +# ARM NEON grid-format decode: what llama.cpp does per 32 weights that we do not + +Scope: the five grid ("codebook") formats on Apple M1 Max, one thread, decode/GEMV +(m=4096, k=14336, one token). Read-only study of +`D:\Work\llama.cpp\ggml\src\ggml-cpu\arch\arm\quants.c` and +`D:\Work\daScript-iquant\modules\dasLLAMA\dasllama\dasllama_gemm_gen.das`. +No file in either tree was modified. + +Measured (us per call, ours vs llama.cpp `test-backend-ops perf -b CPU`): + +| format | ours | llama.cpp | ratio | +|---|---|---|---| +| iq2_xs | 6375 | 3274 | 0.51x | +| iq2_xxs | 6063 | 3437 | 0.57x | +| iq3_xxs | 6735 | 4867 | 0.72x | +| iq3_s | 6089 | 5682 | 0.93x | +| iq2_s | 4932 | 4917 | 1.00x | + +--- + +## 0. Ground facts established first + +**The M1 takes the plain `__ARM_NEON` + dotprod path; there is no i8mm arm for any of the +five.** `__ARM_FEATURE_MATMUL_INT8` appears in `arch/arm/quants.c` only at lines 302, 315, +595, 608, 1155, 1168, 2336, 2360, 2569, 2966, 2984, 3175 - all inside the q4_0/q8_0/q4_K/q5_K +family. The five grid dots start at 3631 (iq2_xxs), 3693 (iq2_xs), 3767 (iq2_s), 3864 +(iq3_xxs), 3926 (iq3_s) and each has exactly two arms: `#if defined(__ARM_NEON)` and the +scalar generic fallback. `ggml_vdotq_s32` is the native `vdotq_s32` when +`__ARM_FEATURE_DOTPROD` is set (`ggml/src/ggml-cpu/ggml-cpu-impl.h:319`), else a +`vmull_s8`+`vpaddlq_s16` emulation (`ggml-cpu-impl.h:310-316`). Our stamped M1 profile +records `"features": "neon;dotprod;fullfp16;lse"` +(`modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json`, provenance block) - +dotprod yes, i8mm no. So both sides run the same primitive: `sdot`. + +**Grid table element widths** (`ggml/src/ggml-common.h`): + +| table | line | element | entries | bytes | weights per entry | +|---|---|---|---|---|---| +| `iq2xxs_grid` | 560 | `uint64_t` | 256 | 2048 | 8 | +| `iq2xs_grid` | 627 | `uint64_t` | 512 | 4096 | 8 | +| `iq2s_grid` | 758 | `uint64_t` | 1024 | 8192 | 8 | +| `iq3xxs_grid` | 1017 | `uint32_t` | 256 | 1024 | 4 | +| `iq3s_grid` | 1052 | `uint32_t` | 512 | 2048 | 4 | +| `ksigns_iq2xs` | 513 | `uint8_t` | 128 | 128 | (7-bit code -> sign byte) | +| `ksigns64` | 524 | `uint64_t` | 128 | 1024 | (7-bit code -> 8 x 0x00/0xFF) | +| `keven_signs_q2xs` | `arch/arm/quants.c:3595` | `int8_t[1024]` | 128 x 8 | 1024 | (7-bit code -> 8 x +/-1) | + +`keven_signs_q2xs` is the ARM-only table: 128 entries of eight `+1`/`-1` bytes, read as +`uint64_t` through `signs64` (`quants.c:3646, 3708, 3879`). It is a **pre-expanded +/-1 +vector**, so applying signs is one `vmulq_s8` with zero bit manipulation. This single table +is the largest structural difference in the whole comparison. + +**Our shape on NEON.** `setup_tile_emit` pins `te.width = 128` on the sdot leg +(`dasllama_gemm_gen.das:1900`), `te.rv = width/32 = 4`, `te.rq = interleave/rv` +(1901-1902). The stamped `mr` is 8 for all five formats on ARM +(`arm-neon.tune-defaults.json`: `iq2sq8_tile_gen`..`iq3xxsq8_tile_gen` all `"mr8"`), so +rv = 4 rows per 16-byte vector, rq = 2 vectors per 8-row group. `sdot_lane` +(`dasllama_gemm_gen.das:309-316`) builds the **indexed** form: the activation is bitcast to +`v4i32`, one dword lane splatted, and the backend folds that into `sdot v,v,v[lane]`. So one +sdot = 4 rows x 4 weights = 16 byte-products. + +**Both sides issue the same number of sdot per weight.** 32 weights x 1 row = 32 +byte-products = 2 sdot for llama.cpp (`quants.c:3747-3750` etc.). Ours: per block per qd, +`for j in 0..3` x lo/hi = 8 sdot covering 4 rows x 32 weights = 128 byte-products, i.e. +2 sdot per row per 32 weights. **The dot is at exact parity. Everything below is the +surrounding work.** + +**Our GEMV re-uses nothing.** `kq_gemv_gen_impl` drives `emit_slice(te, sa, ghead, 0, 1, "")` +(`dasllama_gemm_gen.das:2880`) - tokCount = 1. Every decoded weight is consumed by exactly +one sdot. The alloca panel (`LLVMBuildAlloca(..., 256 * te.interleave, "iq3s.panel")`, line +2845 - 2048 bytes at mr=8) therefore buys **no reuse whatsoever in decode**; it exists only +to separate the rolled gather loop from the unrolled dot body. The prefill/tile path is the +one that benefits, and there the panel is produced once per group by +`unpack_kq_panel_grp` (`dasllama_repack.das:953`) and amortized over 4 tokens - which is +exactly why every prefill tile is 1.9x-15x ahead while decode is behind. + +--- + +## 1. Per-format op budget, side by side + +Counting is per **one row x 32 weights** (llama.cpp's natural unit; ours divided down from +the 4-rows-per-vector shape). "mem" = load/store uops, "NEON" = vector ALU uops, "int" = +scalar integer ALU uops. + +### 1.1 llama.cpp + +**iq2_xxs** (`quants.c:3660-3680`, body covers 64 weights) +- `ggml_vld1q_s8_x4(q8)` (3662): 4 x 16B activation loads / 64w = **2 mem** +- `memcpy(aux32, q2, 16)` (3663): 2 x `ldr x` / 64w = **1 mem** +- grid: 8 x `vld1_s8(iq2xxs_grid + aux8[i])` (3664-3667), paired by `vcombine_s8` into + `ldr d` + `ld1 {v.d}[1]` = **4 mem** + 4 byte-extracts = **4 int** +- signs: 8 x `vld1_s8(signs64 + ((aux32[k] >> s) & 127))` (3668-3671) = **4 mem** + + 4 `ubfx` = **4 int** +- `vmulq_s8` x4 (3672-3675) = **2 NEON** +- `ggml_vdotq_s32` x4 (3676-3677) = **2 NEON** +- epilogue `vaddvq_s32` + float scale (3678-3679) ~ **3 mixed** + +**Total: 11 mem, 4 NEON, 8 int, ~3 epilogue ~ 26 uops.** + +**iq2_xs** (`quants.c:3733-3753`, body covers 64 weights) +- q8 (3734) = **2 mem** +- grid: 8 x `vld1_s8(iq2xs_grid + (q2[i] & 511))` (3735-3738) = 4 index `ldrh` + 4 `and` + + 4 grid `ldr d` (the `vcombine` pair is `ldr d` + `ld1 {v.d}[1]`, so 8 half-loads per + 64w = 4 per 32w) -> **8 mem + 4 int** +- signs: 8 x `vld1_s8(signs64 + (q2[i] >> 9))` (3739-3742) = **4 mem + 4 int** (the + `ldrh` is shared with the grid lookup) +- `vmulq_s8` x4 (3743-3746) = **2 NEON** +- `vdotq` x4 (3747-3750) = **2 NEON** +- `vpaddq_s32` x3 + `vmlaq_s32` (3751-3752) = **2 NEON** +- per-superblock scale prep (3719-3729) amortized over 8 blocks ~ **2** + +**Total: 14 mem, 6 NEON, 8 int ~ 28-30 uops.** + +**iq2_s** (`quants.c:3808-3848`) +- q8 (3809) = **2 mem** +- grid: 8 x `vld1_s8(iq2s_grid + (qs[i] | ((qh[..] << s) & 0x300)))` (3810-3817) = 4 `ldrb` + + 4 grid `ldr d` = **8 mem**, plus `<<`/`&`/`|` x3 per index = **12 int** +- signs are **plane bits**, expanded with the mask1/mask2 shuffle (3820-3824, 3829-3833): + per 32 weights = 1 `vdupq_n_u32` + 2 `vqtbl1q_u8` + 2 `vandq` + 2 `vceqq` = **7 NEON** +- `vorrq_u8(vs, m1)` x2 + `vmulq_s8` x2 (3826-3827) = **4 NEON** +- `vdotq` x2 (3840-3843) = **2 NEON** +- 2 x (`vaddvq_s32` + nibble scale extract + `madd`) (3845-3848) ~ **6 mixed** + +**Total: 10 mem, 13 NEON, 12 int ~ 37 uops.** llama.cpp's most expensive of the five, and +indeed its slowest but one. + +**iq3_xxs** (`quants.c:3894-3912`) +- q8 (3894) = **2 mem**, `memcpy(aux32, gas, 8)` (3895) = **0.5 mem** +- grid: `ggml_vld1q_u32(iq3xxs_grid[q3[0]], .., [q3[3]])` (3896-3899). That macro is a + plain brace initializer on non-MSVC (`ggml-cpu-impl.h:84`), so clang emits 4 index `ldrb` + + 4 `ldr w` + 3 lane inserts per q register, 4 registers per 64 weights -> + **16 mem + 12 int per 64w = 8 mem + 6 int per 32w** +- signs: 8 x `vld1_s8(signs64 + ...)` (3900-3903) = **4 mem + 4 int** +- `vmulq_s8` x4 (3905-3908) = **2 NEON**, `vdotq` x4 (3909-3910) = **2 NEON** +- epilogue (3911-3912) ~ **4 mixed** + +**Total: 14.5 mem, 4 NEON, 10 int ~ 40 uops.** + +**iq3_s** (`quants.c:3983-4022`) +- q8 (3983) = **2 mem** +- index vector: `vld1q_u8(qs)` (3985) then `vmovl_u8`/`vshlq_u16`/`vandq`/`vorrq` + (3986, 3991) = **0.5 mem + ~6 NEON per 32w** +- **the `vec_index_t` union (3941-3944)**: `idx.vec_index` is written as a vector and then + read back as `idx.index[0..7]` scalars (3987-3990, 3992-3995). That is a store-forwarding + round trip through the stack - 1 vector store + 8 `ldrh` per half, **~9 mem per 32w** +- grid: 8 x `ldr w` + 6 lane inserts per 32w = **8 mem + 6 int** +- signs: the mask1/mask2 expansion (3998-4002, 4007-4011) = **7 NEON**, plus `vmulq_s8` x2 + (4004-4005, 4015-4016) = **2 NEON** +- `vdotq` x2 (4018-4019) = **2 NEON**, epilogue (4021-4022) ~ **3** + +**Total: ~19.5 mem, ~17 NEON, ~6 int ~ 49 uops.** llama.cpp's slowest of the five, matching +its 5682 us. + +### 1.2 Ours + +Two loops per superblock, both walked per row group of mr = 8: + +**(A) `emit_kq_gather` (`dasllama_gemm_gen.das:1451-1486`)** - a rolled loop over the mr +rows; each iteration emits **64 straight-line dword decodes** (`for blk in 0..7`, +`for l in 0..3`, `for h in 0..1`, line 1470-1477), each decoding **4 weights of one row** +and storing them with `LLVMBuildStore` into the panel at `panel_group_off` (1443-1449, +1478-1479). + +**(B) the dot body inside `emit_block_iq4xs` (1573-1712)** - per block: 2 activation +`v16i8` loads shared across all 8 rows (1620-1621); per (blk, qd) one `kq_sign_bytes` +(1632); per (blk, j, qd) two panel `v16i8` loads (1642-1643), two `apply_sign_col` +(1645-1646), two `dot_lane` (1659-1662). + +Per **one row x 32 weights**, gather side: + +| format | decode site | mem uops | int uops | +|---|---|---|---| +| iq2_xs | `decode_iq2xs_dword` 1358-1367 | 2 column `ldr w` (`wg_col_byte` 1308-1315) + 4 grid `ldr x` (`grid_pair_half` 1324-1330) + **8 panel `str w`** = **14** | 12 byte extracts + 8 `orr/lsl` for the u16 + 4 `and #511` + 4 `lsr #32` = **28** | +| iq2_xxs | `decode_iq2xxs_dword` 1369-1373 | 4 `ldrb` (`wg_byte` 1293-1296) + 4 grid `ldr x` + **8 `str w`** = **16** | ~8 | +| iq2_s | `decode_iq2s_dword` 1347-1356 | 1 qh `ldrb` + 4 qs `ldrb` + 4 grid `ldr x` + **8 `str w`** = **17** | ~16 | +| iq3_xxs | `decode_iq3xxs_dword` 1342-1344 | 2 column `ldr w` + 8 grid `ldr w` (`grid_word` 1317-1321) + **8 `str w`** = **18** | ~12 | +| iq3_s | `decode_iq3s_dword` 1332-1340 | 1 qh `ldrb` + 8 qs `ldrb` + 8 grid `ldr w` + **8 `str w`** = **25** | ~24 | + +Per **one row x 32 weights**, dot side (shared shape, divided by 4 rows per vector): + +- panel loads: 8 x `ldr q` / 4 rows = **2 mem** +- activation: 2 x `ldr q` / 8 rows = **0.25 mem** +- `apply_sign_col` (1416-1429): the `idx` shuffle depends only on `l`, so 4 distinct `tbl` + per (blk, qd) plus 8 x (`and` + `cmpeq` + `eor` + `sub`) = 36 NEON / 4 rows = **9 NEON** +- `sdot`: 8 / 4 rows = **2 NEON** +- epilogue (sub-scale load, `mul`, `add`, 1683-1706) ~ **1.5 NEON** +- `kq_sign_bytes` (1378-1414): + - **iq2_s (kq 23) and iq3_s (kq 33)**: the plane carries the sign bytes, so it is + **one `ldr q`** (1385-1388) = **0.25 mem, 0 NEON** + - **iq2_xs (24), iq2_xxs (25), iq3_xxs (34)**: 1-2 `ldr q` plus, for each of 4 sign + groups, code extraction + a **3-step parity chain** (`lshr`/`xor` x3, lines 1404-1406) + + `and`/`shl`/`orr` + the byte pack = ~13 NEON x 4 = **53 NEON / 4 rows = 13.25 NEON** + +**This is the single biggest asymmetry.** llama.cpp never computes a parity: it indexes +`keven_signs_q2xs` (`quants.c:3595`), a table whose entries already carry the eighth, +parity-implied sign. We recompute the eighth sign in registers, for every sign group, of +every block, of every superblock, of every row. + +### 1.3 A port model that fits the measurements + +M1 Firestorm: 8-wide issue, ~3 load units, ~2 store units (~3 memory uops/cycle sustained), +4 NEON pipes, ~6 integer ALU pipes, 3.22 GHz. Modelling each loop as +`max(mem/3, NEON/4, int/6)` cycles per row per 32 weights: + +| format | gather cyc | dot cyc | total cyc | predicted us | measured us | error | +|---|---|---|---|---|---|---| +| iq2_s | 5.7 | 3.2 | 8.9 | 5070 | 4932 | +3% | +| iq2_xs | 4.7 | 6.45 | 11.15| 6354 | 6375 | 0% | +| iq2_xxs | 5.3 | 6.45 | 11.75| 6696 | 6063 | +10% | +| iq3_s | 8.3 | 3.2 | 11.5 | 6553 | 6089 | +8% | +| iq3_xxs | 6.0 | 6.45 | 12.45| 7094 | 6735 | +5% | + +(predicted us = cycles x 1.835e6 row-32w units x 1e6 / 3.22e9; 4096 x 14336 / 32 = 1.835e6.) + +**The model tracks all five within 10%, which means our kernel is genuinely port-saturated.** +The same model applied to llama.cpp under-predicts by 20-50% (iq2_xs: 4.67 cyc -> 2661 us vs +3274 measured; iq3_s: 6.5 cyc -> 3705 vs 5682), because llama.cpp's serial +index-load -> grid-load chains and per-32 float accumulation leave it latency-bound with +issue slack. Practical consequence: **op count is our currency and only our currency; we +cannot buy time back with better scheduling, and llama.cpp's measured times are an upper +bound on what its op mix actually costs.** + +The model also reproduces the *ranking* exactly, and it decomposes into two independent +additive penalties: + +- **+3.25 cyc/row/32w** if the format's signs are ksigns codes rather than plane bytes + (iq2_xs, iq2_xxs, iq3_xxs). +- **+2.6 cyc/row/32w** if the grid is `u32` (4 weights/entry) rather than `u64` + (8 weights/entry), because the gather issues twice the grid loads (iq3_s, iq3_xxs). + +iq2_s pays neither and sits at 1.00x. iq3_s pays only the second and sits at 0.93x. The +three at 0.51x-0.72x are exactly the three that pay the parity chain. **The 0.51x/0.57x +outliers are not "ARM is hard"; they are one identifiable code path.** + +--- + +## 2. The deltas, ranked by decode payoff on M1 + +Each entry is tagged **emitter change** / **repack change** / **seat**, with its x86 +exposure. Zen2 is at 0.99x-1.32x on these five, so nothing here may touch the AVX2 lattice +without a re-measure. + +### D1 - the gather decodes one dword per scalar step and round-trips 32 B/row/block through an alloca + +**Payoff: the largest. Tag: emitter change (GEMV-only, sdot-leg-only). x86: none.** + +`emit_kq_gather` decodes 4 weights at a time (`decode_kq_dword`, 1431-1438) and stores them +as a 4-byte `str` (1478-1479); the dot body then reloads them as `v16i8` +(1642-1643). Per row per 32 weights that is **8 stores + 2 loads = 10 of the ~14-25 memory +uops** in the gather. Since tokCount = 1 in decode (line 2880), those 2 KB are written and +read back once, for nothing. + +Worse, the 4-byte granularity is *forced* by the panel's element order. For the u64-grid +formats one grid entry is 8 consecutive weights `l*8 .. l*8+7`, but +`panel_group_off` (1443-1449) sends `e = l*8` and `e = l*8+4` to different dword groups +`mr*4 = 32` bytes apart (the same split is visible in the tile's unpack: +`dasllama_repack.das:896-901`, `d1`/`d2` computed separately, then two 4-byte +`wp1[0]`/`wp2[0]` stores). So a single 8-byte grid load costs two 4-byte stores plus a +`lsr #32` to split it (`grid_pair_half`, 1324-1330). + +llama.cpp never splits: `vcombine_s8(vld1_s8(grid+i), vld1_s8(grid+j))` (3735) puts two +whole u64 entries - 16 consecutive weights of one row - into one q register with two loads +and no store at all. + +### D2 - the three ksigns formats synthesize the eighth sign bit with a 3-step parity chain, per group, per block, per row + +**Payoff: 3.25 cyc/row/32w, ~29% of iq2_xs's time. Tag: emitter change (if D1 lands) or +repack change (standalone). x86: shared code, must re-measure zen2.** + +`kq_sign_bytes` lines 1400-1412: `code` -> `par = code ^ (code>>4)`, `par ^= par>>2`, +`par ^= par>>1`, `sb = code | ((par&1)<<7)`, then a 4-way byte pack. 13 vector ops per sign +group, 4 groups per block, on `<4 x i32>` - 53 NEON uops per (4 rows, 32 weights). + +llama.cpp: `vld1_s8(signs64 + code)` - one load of a pre-expanded +/-1 vector +(`quants.c:3739-3742`, table at 3595), then one `vmulq_s8`. The parity is baked into the +table. + +Note the comment drift here: lines 1484-1485, 1507, 1529 and 1549 all describe "the shared +ksigns table" and "the shared smask table" as emitted globals, but the `*_emit_globals` +functions only create the grid global - no ksigns or smask constant is emitted, and +`kq_sign_bytes` reads none. Those four comment fragments are stale. + +### D3 - the u64 grid's 8 weights are decoded as two separate 4-weight steps + +**Payoff: it is what makes D1's 8 stores 8 instead of 4, and it doubles the index work for +iq2_xs. Tag: emitter change. x86: same code path, gated by the same `gather` flag.** + +`grid_pair_half` (1324-1330) issues one 8-byte load and then `trunc`s the low or high half +per call. LLVM's CSE folds the two calls onto one load, but the *consumers* are two +independent 4-byte stores at unrelated panel offsets, so the pair can never become a single +8-byte store. + +### D4 - the packed plane's 4-byte column layout costs a dword load plus shift and mask per index byte + +**Payoff: ~2 cyc/row/32w on iq2_xs specifically. Tag: repack change or seat. x86: the +current shape was chosen ON x86.** + +`wg_col(mr, pos) = (pos/4)*mr*4 + pos%4` (1304) interleaves rows in 4-byte columns, so +`wg_col_byte` (1308-1315) reads a dword and shifts the byte out. For iq2_xs the 16-bit qs +word is then rebuilt from two such bytes (`decode_iq2xs_dword`, 1358-1367: `qlo | (qhi<<8)`), +where llama.cpp does one `ldrh` (3735). That is 12 extraction + 8 reassembly ops against +llama.cpp's 4 loads. + +The doc-comments on 1342 and 1358 record the measurements that chose this shape +("the column read measured 7862 -> 6602 us", "6501 -> 4702 us; one u16 load measured slower +than two bytes"). Those figures are from the x86 development box, and the same comment on +1342 already admits the column read *lost* for iq3s/iq2s/iq2xxs. **The decode shape is +currently one global decision made on one ISA.** It should be a seat. + +### D5 - the sign application is a 5-op masked negate rather than one multiply + +**Payoff: small on its own (~1 cyc/row/32w), large once D1 lands. Tag: emitter change.** + +`apply_sign_col` (1416-1429): `tbl` broadcast of the row's sign byte, `and` with a per-lane +bit constant, `cmpeq`, `eor`, `sub`. Against llama.cpp's `vmulq_s8` with a table-loaded ++/-1 vector. Our form costs 4 ALU ops per weight vector plus a shared `tbl`; llama.cpp's +costs 1 multiply plus the 2 loads already counted. + +But this is a *consequence*, not a cause: our 16-byte vector holds **4 different rows**, so +a +/-1 vector for it would need four separate table entries and four lane inserts. The +masked negate is the correct choice given the current layout. It is only fixable by changing +the layout (D1/section 3a). + +Note the cost is already close to llama.cpp's for iq2_s/iq3_s: our 9 NEON/row/32w against +llama.cpp's 11 for the mask1/mask2 expansion (`quants.c:3820-3827`). This is precisely why +iq2_s measures 1.00x. + +--- + +## 3. The three proposals evaluated + +### (a) A width-128 gather composing u64 grid entries as 64-bit lanes: 2 rows x 8 weights per vector + +**Verdict: this is the fix. Adopt it.** + +Layout: vector `W = [ row A weights e..e+7 | row B weights e..e+7 ]`, one 16-byte register +from two 8-byte grid loads (`ldr d` into lane 0, `ld1 {v.d}[1]` into lane 1 - in LLVM, a +load plus an `insertelement` into a `<2 x i64>`). + +The dot then uses the **non-indexed** `sdot Vd.4S, Vn.16B, Vm.16B`, whose lane i is +`Vn[4i..4i+3] . Vm[4i..4i+3]`. Feed it `X = [ x(e..e+3) | x(e+4..e+7) | x(e..e+3) | +x(e+4..e+7) ]`, i.e. the 8 activation bytes replicated - exactly `ld1r {v.2d}, [xp]`, one +instruction, no lane-splat shuffle at all. Then: + +- lane 0 = row A . x(e..e+3), lane 1 = row A . x(e+4..e+7) +- lane 2 = row B . x(e..e+3), lane 3 = row B . x(e+4..e+7) + +Accumulate all 4 element-groups of a block into **one** `<4 x i32>` per row pair; at block +end, one `addp` of two row-pair accumulators yields `[rowA, rowB, rowC, rowD]` - **the exact +`<4 x i32>` shape `a[]` already has** (line 1659-1662), so the entire scale/fold epilogue +(1683-1711) is untouched. `pairwise_add_i32` (909-917) already emits that `addp`. + +Products per sdot: 2 rows x 8 weights = 16, identical to today's 4 x 4. **No dot regression.** + +Same intrinsic as today (`llvm.aarch64.neon.sdot`, `<4 x i32>`/`<16 x i8>`, declared at +1911-1918) - only the operand construction changes, so no new decline rail. + +Projected budget per **2 rows x 32 weights** for iq2_xs, keeping the panel: + +| | uops | +|---|---| +| index loads (a row pair's 4+4 index bytes are contiguous in the column -> one `ldr x`) | 2 mem | +| grid: 8 u64 loads | 8 mem | +| signs: 8 loads from a +/-1 table, indexed by the 7-bit code | 8 mem | +| `vmulq_s8` x4 | 4 NEON | +| panel: 4 x 16-byte stores | 4 mem | +| dot: 4 panel loads + 4 sdot + 1 `ld1r` | 5 mem, 4 NEON | +| index extraction | ~16 int | + +mem 27 -> 9 cyc, NEON 8 -> 2 cyc, int 16 -> 2.7 cyc. **9 cyc per 2 rows = 4.5 cyc/row/32w** +against today's 11.15. At the fitted 572 us/cycle-unit that is **iq2_xs ~2570 us, i.e. +1.27x AHEAD of llama.cpp's 3274**. Even a 50% miss lands at parity. + +If the gather is *fused* into the dot body (legitimate in decode, where tokCount = 1 and the +panel buys nothing), the 4 stores and 4 loads vanish: ~19 mem -> 6.3 cyc per 2 rows = +**3.2 cyc/row/32w, ~1810 us**. Cost: the superblock body unrolls to 4 row-pairs x 32 +element-groups x ~6 ops ~ 770 instructions, which is actually *smaller* than today's +combined dot body (~900) plus gather body. Worth trying second. + +Notes and caveats: + +- **The +/-1 sign table replaces D2 entirely.** Under this layout the sign vector for a row + pair is two 8-byte loads from a `keven_signs`-equivalent global indexed by the 7-bit code - + no parity chain, no plane change, no repack. D2 becomes unnecessary if (a) lands. +- **For iq2_s / iq3_s** (signs are plane *bits*, not codes) emit a 256-entry x 8-byte +/-1 + table indexed by the raw sign byte (2 KB private constant, same shape as the grid globals + at 1273-1291) so all five formats share one sign path. +- **For the u32-grid formats** (iq3_xxs, iq3_s) a row's 8 weights are two grid entries, so + building the half costs 2 loads (`ldr s` + `ld1 {v.s}[1]`) instead of 1 - 16 grid loads per + (2 rows, 32w) instead of 8. Projected ~5 cyc/row/32w, still ~2.3x better than today and + ahead of llama.cpp, which pays the same 8 u32 loads per row plus 6 lane inserts + (`quants.c:3896-3899`, `ggml-cpu-impl.h:84`). +- **Blast radius is small and ARM-only.** The panel is the GEMV's private alloca (line 2845), + the new shape lives entirely inside the `if (gather)` branch (line 1581), and the whole + thing is gated on `te.dotKind == DOT_SDOT`. **No plane change, no repack change, no tile + change, no x86 exposure.** Prefill (already 1.9x-15x ahead) is untouched. +- **Make it a seat for one tuning round.** Add a perm token (`gather="pair"` vs the current + `"dword"`) parsed in `parse_perm` (101-125) so the M1 sidecar proves it per format before + it becomes the default. The existing seats are one string per format + (`iq2xsq8_tile_gen => "mr8"`), so the token composes with `mr8` without disturbing x86's + `dot_maddubs_width256_mr8`. + +### (b) ARM gather via vector table lookup (`vqtbl`) + +**Verdict: impossible for the grid. Not worth an experiment.** + +NEON `TBL`/`TBX` index into 1 to 4 *consecutive V registers* - a 16, 32, 48 or 64-byte +table, with out-of-range indices producing zero. The smallest grid here is `iq3xxs_grid`, +256 x `uint32_t` = **1024 bytes** (`ggml-common.h:1017`) - 64 V registers, 16x the ISA +maximum. `iq2xxs_grid` is 2048 bytes (`ggml-common.h:560`). There is no NEON gather +instruction of any kind, and the M1 has no SVE (profile features string: +`neon;dotprod;fullfp16;lse`). So the grid must be read with scalar loads on this ISA, and +llama.cpp does exactly that (`vld1_s8`/`ldr w` per entry) for the same reason. + +Where `tbl` *is* the right tool, and where both sides already use it: + +- the 16-entry nibble codebook for iq4_nl/iq4_xs - we already emit `llvm.aarch64.neon.tbl1` + (`dasllama_gemm_gen.das:1929-1936`, `lut_lookup` 632-642) +- broadcasting a sign byte across its four lanes - `apply_sign_col`'s constant + `shufflevector` (1421-1422) lowers to `tbl`, and llama.cpp's `mask1` shuffle + (`quants.c:3822`, `4000`) is the same instruction + +No format in this set has a grid small enough for a `vqtbl4q` sub-table. + +### (c) Sign application: `vmulq_s8` with a table-derived sign vector vs our and+cmpeq+xor+sub + +**Verdict: our masked negate is not the defect. The defect is the sign *source*, and the +multiply form is unreachable without (a).** + +Cost accounting, per one row x 32 weights: + +| scheme | ops | +|---|---| +| llama.cpp, ksigns formats (`quants.c:3739-3746`): 4 `vld1_s8` + 2 `vmulq_s8` | 4 mem + 2 NEON | +| llama.cpp, plane-bit formats (`quants.c:3820-3827`): dup + 2 tbl + 2 and + 2 ceq + 2 orr + 2 mul | 11 NEON | +| ours, `apply_sign_col` (1416-1429): 4 tbl + 8 x (and, cmpeq, eor, sub), / 4 rows | 9 NEON | +| ours, `kq_sign_bytes` ksigns arm (1400-1412): 53 NEON / 4 rows | 13.25 NEON | +| ours, `kq_sign_bytes` plane arm (1385-1388): 1 load / 4 rows | 0.25 mem | + +So our *application* (9) already beats llama.cpp's bit expansion (11) and is within reach of +its multiply form (2 NEON + 4 mem). The 13.25 NEON of *code-to-byte synthesis* is the whole +problem, and it exists only because the format stores a 7-bit code where iq2_s and iq3_s +store a byte. + +Two ways to kill it: + +1. **Under (a)**: index a +/-1 table with the 7-bit code and multiply. Zero synthesis, zero + plane change. This is llama.cpp's move, and it is only available once a vector holds + whole 8-weight runs of a *single* row (a 4-row-interleaved vector would need 4 separate + table entries plus 4 lane inserts, which is why the masked negate was correct until now). +2. **Standalone (repack change, if (a) is deferred)**: store the expanded 8-bit sign byte in + the plane instead of the 7-bit code, turning iq2_xs/iq2_xxs/iq3_xxs into the iq2_s/iq3_s + shape. + - **iq2_xxs**: the aux32 column already holds 4 x 7-bit codes plus a 4-bit `ls` in 4 + bytes, and the scale already lives in the scale plane (`IQ2XXS_SSB` comment, + `dasllama_kqformat.das:118`), so 4 sign bytes fit in the same 4 bytes. + **Size-neutral** (`IQ2XXS_QSB` stays 64, line 117). + - **iq3_xxs**: identical - 8 blocks x 4 aux bytes -> 8 x 4 sign bytes. + **Size-neutral** (`IQ3XXS_QSB` stays 96, line 111). + - **iq2_xs**: the 9-bit index does not fit a byte. Restructure to iq2_s's shape - + 32 index low bytes + 4 high-bit bytes + 32 sign bytes = 68 B against today's 64 + (`IQ2XS_QSB`, line 116). **+6%.** + Projected under this alone: iq2_xs 6375 -> ~4520 (0.72x), iq2_xxs 6063 -> ~4390 (0.78x), + iq3_xxs 6735 -> ~4980 (0.98x). Real, but short of parity for the iq2 pair, which is why + (a) is the recommendation and this is the fallback. + + **x86 caution for option 2**: it changes the on-disk plane, the repack, the tile unpack + (`dasllama_repack.das:874-949`) and both ISAs. The +6% on iq2_xs is affordable on + bandwidth grounds - the decode moves 16 MB per call in 6375 us = 2.5 GB/s against M1 + Max's tens of GB/s single-core, so both implementations are ~15-25x off the memory roof + and firmly compute-bound - but zen2's iq2_xs must be re-measured before it lands. iq2_xxs + and iq3_xxs being size-neutral carry no bandwidth risk on either ISA. + +--- + +## 4. Recommended order + +1. **(a) as a seat**, u64-grid formats first (iq2_xs, iq2_xxs, iq2_s), panel retained, + signs from an emitted +/-1 table. Emitter-only, GEMV-only, sdot-leg-only, no x86 + exposure. Model says 0.51x -> ~1.27x on iq2_xs; measure before believing. +2. **(a) extended to the u32-grid formats** (iq3_xxs, iq3_s) with the 2-loads-per-half + build. Model says ~5 cyc/row/32w, comfortably ahead. +3. **Fuse the gather into the dot body** (drop the panel in decode only) once (a) is + stamped. Model says a further ~30%. Watch generated code size. +4. **Make the decode shape a seat** regardless (D4): the column-read-vs-byte-read choice at + lines 1342 and 1358 was measured on x86 and is currently binding on ARM. +5. **Only if (a) is abandoned**: the plane sign-byte repack for iq2_xxs and iq3_xxs + (size-neutral, safe on both ISAs), and separately iq2_xs (+6%, needs a zen2 re-measure). + +Incidental finding for whoever touches these lines: the "shared ksigns table" and "shared +smask table" described in the comments at `dasllama_gemm_gen.das:1484-1485, 1507, 1529, 1549` +are not emitted by the functions those comments head, and `kq_sign_bytes` reads neither. diff --git a/plans/kernel_parity_research_cpu.md b/plans/kernel_parity_research_cpu.md new file mode 100644 index 0000000000..7f36a319f4 --- /dev/null +++ b/plans/kernel_parity_research_cpu.md @@ -0,0 +1,670 @@ +# CPU grid-quant dot: what llama.cpp does, what we do, and the gap + +Read-only study of five codebook ("grid") formats - IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S - +plus Q2_K as the near-parity control, for the single-token decode shape (GEMV). + +Trees read: + +- `D:\Work\llama.cpp` - `ggml/src/ggml-cpu/arch/x86/quants.c`, `arch/arm/quants.c`, + `quants.c`, `ggml-cpu.c`, `repack.cpp`, `llamafile/sgemm.cpp`, `ggml/src/ggml-common.h` +- `D:\Work\daScript-iquant` - `modules/dasLLAMA/dasllama/dasllama_gemm_gen.das`, + `dasllama_math_gen.das`, `dasllama_repack.das`, `modules/dasLLVM/daslib/llvm_tune.das` + +Nothing was modified in either tree. + +--- + +## 0. The one structural fact that frames everything + +llama.cpp and we solve a *different shape*, and almost every technique difference follows +from that. + +**llama.cpp is one row at a time.** `ggml_vec_dot_iq3_s_q8_K(n, s, bs, vx, bx, vy, by, 1)` +walks one weight row against one activation row. Thirty-two consecutive weights of that one +row form one 256-bit register. Because the activation vector is *the same 32 bytes* the row +is being dotted against, llama.cpp is free to move the per-weight sign onto the activation. +Each row ends with a horizontal float reduction (`hsum_float_8`, x86 quants.c:2713) or, on +ARM, with a cross-lane `vaddvq_s32` per 32-weight sub-block (arm quants.c:3678). + +**We are `mr` rows at a time, activation broadcast.** `setup_tile_emit` +(gemm_gen.das:2123-2141) sets `rv = width/32` and `rq = mr/rv`; one weight vector holds +**4 bytes from each of `rv` different output rows**. `dot_lane` (gemm_gen.das:328-366) splats +one dword-group of the activation across every lane and dots. For the seated x86 perm +(`width = 256, mr = 8`) one `<32 x i8>` weight vector = 8 rows x 4 weights; for AVX-512 +(`width = 512, mr = 16`) it is 16 rows x 4 weights; on NEON (`width = 128, mr = 8, rq = 2`) +it is two vectors of 4 rows x 4 weights. + +Consequences, both directions: + +- **We win the reduction.** Our accumulator lanes *are* the output rows. There is no + horizontal sum anywhere in the kernel - `emit_block_iq4xs` ends with one `sitofp` + one + FMA per superblock per row-lane (gemm_gen.das:1943-1953). llama.cpp's ARM kernels pay a + `vaddvq_s32` per 32-weight block per row (arm:3678, 3845, 3911, 4021), i.e. 8 cross-lane + reductions per superblock per row on a serial dependency chain. This is almost certainly + why our prefill/tile numbers on M1 are 2.1x-6.4x while decode lags. +- **We cannot put signs on the activation.** The 8 lanes of one weight vector belong to 8 + *different* rows with 8 *different* sign patterns, while the activation dword is shared + across all lanes. Signs must go on the weights. This is a hard consequence of the layout, + not a choice, and it is the part of followup entry 61 that needs correcting (see sec.8). +- **Grid decode cannot stream.** A grid word is 4 or 8 bytes of *one* row. In llama.cpp's + layout, 8 consecutive grid words of one row *are* the next vector, so the composition + `_mm256_set_epi32(grid[i7],...,grid[i0])` writes the register directly. In our layout a + weight vector needs one grid dword from each of 8 different rows - which is *also* exactly + a `set_epi32` of 8 table reads. The composition is available to us; we just do not use it. + Today we round-trip through an `alloca` panel instead. + +--- + +## 1. llama.cpp's mul_mat path selection for these types + +`ggml_compute_forward_mul_mat` (ggml-cpu.c:1254). + +- `src1` (activations) are quantized to the row type's `vec_dot_type` **once**, cooperatively + by all threads, into `params->wdata` (ggml-cpu.c:1321-1355). For all six formats + `vec_dot_type = GGML_TYPE_Q8_K` (ggml-cpu.c:298-366). +- `nrows = 1` for all six (ggml-cpu.c:302, 338, 344, 350, 356, 362), so + `num_rows_per_vec_dot` is 1 (ggml-cpu.c:1435) - no 2-row MMLA fast path, unlike Q4_K/Q6_K + which get `nrows = 2` under `__ARM_FEATURE_MATMUL_INT8` (ggml-cpu.c:313-317, 331-335). +- Work is chunked over rows and dispatched to + `ggml_compute_forward_mul_mat_one_chunk` (ggml-cpu.c:1164), which calls `vec_dot` per row. +- **`from_float` is NULL for all five grid formats** (ggml-cpu.c:337, 343, 348-349, 356, 362) + - these types are never produced on the fly, only read. + +**Repack / "extra buffer type": none of the five grid formats has one.** +`ggml_repack_get_optimal_repack_type` (repack.cpp:4528) covers Q4_0, Q8_0, Q4_K, Q5_K, Q6_K, +MXFP4, IQ4_NL, and **Q2_K** - nothing else. So for IQ2_XXS/IQ2_XS/IQ2_S/IQ3_XXS/IQ3_S, +decode is *always* the per-row `vec_dot` above. + +**Q2_K is the exception and it matters for the control measurement.** repack.cpp:4627-4632 +selects `q2_K_8x8_q8_K` when `ggml_cpu_has_avx512()` and `ne[1] % 8 == 0`; the GEMV entry is +`gemv` (repack.cpp:3976-3984) -> `ggml_gemv_q2_K_8x8_q8_K`. +That is an 8-row-interleaved GEMV - *our* shape. There is also a RISC-V `q2_K_16x1_q8_K` +(repack.cpp:4633-4642). So a Q2_K comparison on zen2 (no AVX-512) races +`ggml_vec_dot_q2_K_q8_K`, and the same comparison on zen4 races the repacked 8x8 gemv. Worth +keeping straight when reading the 1.02x/0.95x control numbers. + +`llamafile_sgemm` (ggml-cpu.c:1293-1318, 1364-1385) covers only F32, BF16, F16, Q8_0, Q4_0, +Q5_0, IQ4_NL (sgemm.cpp:3829, 3893, 3957, 4041, 4078, 4115, 4131) - none of the six. + +--- + +## 2. Grid and sign tables (shared by both trees) + +- `iq2xxs_grid` `uint64_t[256]` (ggml-common.h:560), `iq2xs_grid` `uint64_t[512]` (:627), + `iq2s_grid` `uint64_t[1024]` (:758) - 8 weights per entry. +- `iq3xxs_grid` `uint32_t[256]` (:1017), `iq3s_grid` `uint32_t[512]` (:1052) - 4 weights per + entry. +- **Every grid byte is a small non-negative magnitude**: iq2xxs bytes are `{0x08,0x19,0x2b}` + (ggml-common.h:561-566), iq3xxs `0x04..0x3e` (:1018), iq3s `0x01..0x0f` (:1053). Two + things follow: (i) the magnitudes are legal `u8` operands for `VPMADDUBSW`/`VPDPBUSD`, and + the worst-case pair sum `2 x 62 x 127 = 15748` cannot saturate `i16`; (ii) no magnitude is + zero, so per-byte two's-complement negation inside a 32-bit word never carries across a + byte lane - a fact both trees rely on (ours explicitly, gemm_gen.das:1477-1479). +- `ksigns_iq2xs` `uint8_t[128]` (ggml-common.h:513): 7 sign bits -> the 8-bit sign byte with + bit 7 set to the parity completion. `kmask_iq2xs[8] = {1,2,4,...,128}` (:509). +- `keven_signs_q2xs` `int8_t[1024]` = 128 x 8 `+/-1` bytes, defined *per arch file* + (x86:2624, arm:3595, loongarch:1577, powerpc:1388). Read as `const uint64_t * signs64` + (x86:2675, arm:3646) so one 64-bit load gives 8 `+/-1` bytes. + +--- + +## 3. IQ2_XXS + +Packing: per 32 weights, four 8-bit grid indices + a 32-bit aux word carrying four 7-bit +sign codes and a 4-bit scale in the top nibble. + +### (a) llama.cpp x86 (AVX2), `ggml_vec_dot_iq2_xxs_q8_K`, quants.c:2660-2714 + +1. Staging: `memcpy(aux32, q2, 16)` into a 16-byte scalar array, aliased as `aux8` + (x86:2677-2678, 2690). Scalar staging of *packed bytes only* - never of decoded weights. +2. Grid compose: `_mm256_set_epi64x(iq2xxs_grid[aux8[3]], ..., aux8[0])` (x86:2691-2692) - + four 64-bit table reads written straight into a ymm. No gather instruction, no store. +3. Signs: `signs64[(aux32[1] >> 21) & 127]` etc. (x86:2693-2696) - four 64-bit reads of + `keven_signs_q2xs` composed into a second ymm of `+/-1` bytes. The parity-completed 8th + sign is *in the table*; no parity arithmetic at runtime. +4. Sign application: `_mm256_sign_epi8(q8_1, s2_1)` (x86:2697-2698) - **applied to the + activation**, one instruction per 32 bytes. Grid magnitudes stay unsigned. +5. Dot: `_mm256_maddubs_epi16(q2_1, q8s_1)` (x86:2699-2700), u8 x s8 -> i16 pairs. +6. Scale: `_mm256_madd_epi16(dot, _mm256_set1_epi16(2*ls+1))` (x86:2703-2704) - the per-32 + scale is folded into the *same* `vpmaddwd` that does the i16->i32 pair reduction. One + instruction does scale and widening together. +7. Accumulate: two independent i32 ymm chains `sumi1`/`sumi2` (x86:2705-2706), added once + per superblock, `cvtepi32_ps` + `fmadd` with `d = x.d * y.d` (x86:2709). The `0.125f` + global factor is applied once at the very end (x86:2713). + +No intermediate weight buffer. Only the 16-byte `aux32` staging array touches memory. + +An SSE/AVX-only fallback exists (x86:2715-2770) using `_mm_set_epi64x` pairs. + +### (b) llama.cpp arm64 (NEON+dotprod), quants.c:3631-3689 + +1. Same 16-byte `aux32` staging (arm:3644-3645, 3663). +2. Grid compose: `vcombine_s8(vld1_s8(iq2xxs_grid + aux8[0]), vld1_s8(... + aux8[1]))` + (arm:3664-3667) - two 8-byte table loads combined into a q-register. NEON's equivalent of + `set_epi64x`. +3. Signs: same shape from `signs64` (arm:3668-3671). +4. Sign application: `vmulq_s8(q2u.val[i], q2s.val[i])` (arm:3672-3675) - **applied to the + weights**, because `sdot` is signedxsigned and there is no `u8xs8` NEON dot. This is the + arch-forced mirror of the x86 choice. +5. Dot: `ggml_vdotq_s32` chained twice per 32 weights (arm:3676-3677). +6. Scale + accumulate: `sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[1] >> 28))` + (arm:3678-3679) - a **cross-lane reduction and a float multiply-add per 32-weight + block**, 8 per superblock. This is the weakest part of llama.cpp's ARM grid kernels. +7. `*s = 0.25f * sumf` (arm:3684). + +### (c) What ours does today + +Seat: `dasllama_math_gen.das` - the `iq2xxsq8_tile_gen` block, four perms only +(`mr=8` NEON sdot; `maddubs/256/8`; `vpdpbusd/256/8`; `vpdpbusd/512/16`), same list as every +other grid format. GEMV generator: `iq2xxs_gemv_gen` -> `kq_gemv_gen_impl(gc, 25)` +(gemm_gen.das:3344). + +- Panel alloca `256 * mr` bytes, `align 32`, one per GEMV call + (gemm_gen.das:3081-3082). +- Per superblock, `emit_iq2xxs_gather` (gemm_gen.das:1749-1806) runs an **`mr`-iteration row + loop**; inside, for each of 8 blocks and 4 sub-lanes: + - the 4-byte aux word is read as **four separate `i8` loads plus three shifts and three + ORs** (gemm_gen.das:1766-1770) - note IQ3_XXS reads the identical field with **one + aligned `i32` load** (gemm_gen.das:1438-1439). The offsets `(2*blk+1)*mr*4 + 0..3` are + contiguous and 4-aligned, so the four-load spelling is pure waste. + - `ksigns` lookup: one `i32` load from the `[128 x i32]` global (gemm_gen.das:1633-1687, + 1775-1776). + - grid: two `i32` loads from the `[1024 x i32]` low/high-word-pair global + (gemm_gen.das:1782-1784). + - sign apply: `w = (gw ^ mw) + (mw & 0x01010101)` where `mw` is an `i32` from a + 16-entry nibble->byte-mask table (gemm_gen.das:1785-1787) - **three GPR ops and two loads + per 4 weights**, i.e. ~40 scalar ops per 32 bytes of weight. + - store: one 4-byte store into the panel (gemm_gen.das:1793-1794). +- The dot then reloads the panel as `<32 x i8>` vectors (gemm_gen.das:1880-1883), takes + `abs_w` to recover the magnitudes it just threw away (gemm_gen.das:1888-1889, + 370-375 - `llvm.abs` -> `VPABSB`), signs the *splatted activation* with `VPSIGNB` + (gemm_gen.das:349-357), then `maddubs` + `pmaddwd` (gemm_gen.das:361-365). +- Scale: per-32 `i8` scale row loaded as `<8 x i8>`, `sext` to `<8 x i32>`, then a **`vpmulld` + per block per row-lane** (gemm_gen.das:1922-1924, 1935). Integer accumulation across the + superblock, one `sitofp` + FMA at the end (gemm_gen.das:1943-1953). The `0.25f` global + factor is pre-baked into the repacked scale row, so it costs nothing at runtime. + +**The memory shape is the wrong way round.** The store address is +`panel + r*4 + (blk*4 + g)*mr*4 + half` (gemm_gen.das:1793); consecutive stores for one row +are `mr*4 = 32` bytes apart, and the reload at +`(blk*16 + j*4)*mr` (gemm_gen.das:1875) is a 32-byte load covering **8 distinct 4-byte +stores from 8 different loop iterations**. A wide load spanning several narrow stores is the +one direction x86 store-to-load forwarding cannot take. llama.cpp does round-trip through +memory - but only for *indices*, and only in the safe direction (sec.6c): one wide store, then +narrow fully-contained loads. + +Rough uop budget per 32-byte weight vector (mr = 8, AVX2): +ours ~ 8x(2 loads + 3 ALU + 1 store) + 1 reload + `abs` + `psign` ~ 51; +a register-composed form ~ 9 (`vpinsrd` with folded grid load) + 9 (mask) + `vpor` + +`vpsignb` ~ 20. + +### (d) Deltas, ranked + +| # | Delta | Candidate seat | Why / estimated payoff | +|---|---|---|---| +| 1 | **Delete the panel.** Compose the 8 grid dwords of one weight vector directly with `insertelement` (the `set_epi32` shape) instead of storing them and reloading. | `tune_perm(dot="maddubs", width=256, mr=8, gather="reg", requires="avx2")` and the `vpdpbusd` twins | Removes 512 scattered 4-byte stores + 64 wide reloads per superblock per row-group, and the store-forwarding hazard. Biggest single item; ~2.5x fewer decode uops. | +| 2 | **Sign in vectors, not GPRs.** Compose a second vector of the 0x00/0xFF byte masks and apply `vpor(mask,1)` + `vpsignb` to the activation, keeping the raw grid vector as the unsigned `maddubs` operand. Deletes both the 3-GPR-op-per-dword negate *and* `llvm.abs`. | folded into `gather="reg"`, or split as `psign="mask"` | ~24 GPR ops -> 2 vector ops per 32 bytes; also removes `VPABSB` from the inner loop. | +| 3 | **One `i32` load for the aux word.** Replace the four `i8` loads + 3 shifts + 3 ORs at gemm_gen.das:1766-1770 with the single aligned `i32` load IQ3_XXS already uses (gemm_gen.das:1438-1439). | no seat needed - a straight bug-shaped fix | 6 uops x 8 blocks x mr rows per superblock, for free. Do this regardless. | +| 4 | **Bake the parity-completed sign byte at repack time.** The aux word is 4x7 sign bits + a 4-bit scale, and the scale already moved to the scale row (gemm_gen.das:1746-1748) - so 4 full sign *bytes* fit in the same 4 bytes. Deletes the `ksigns` load entirely and makes the sign bits plane-resident like IQ3_S's. | repack-side; pairs with delta 2 | Removes one dependent table load per 8 weights. Zero size change for this format. | +| 5 | **Seat `vpdpbssd`.** The panel bytes are already signed and the activation is signed; `dot_lane` takes that path with no `abs` and no `psign` (gemm_gen.das:344-347, 370-372). Currently unseated for every grid format. | `tune_perm(dot="vpdpbssd", width=256, mr=8, requires="avxvnniint8")` | One-line seat, zero generator change. Only helps AVX-VNNI-INT8 hosts (not zen2/zen4). | +| 6 | **Seat `bias=128`.** Under `gather="reg"` the biased byte is free - `((mag^mask) + (mask&0x01010101)) ^ 0x80808080` - and `bias128` drops `abs` and `psign` on VNNI hosts. Blocked today: `setup_tile_emit` forces `te.bias = 0` whenever `te.kq != 0` (gemm_gen.das:2136). | `tune_perm(dot="vpdpbusd", width=256, mr=8, bias=128, gather="reg", requires="avxvnni\|avx512vnni")` | Needs the `te.bias` clamp lifted for panel formats. Not valid under `maddubs` - `(w+128)xx` pair sums saturate `i16`. | +| 7 | **Fold the scale into `pmaddwd`.** llama.cpp gets scale + widening in one `vpmaddwd` (x86:2703). We cannot broadcast a scalar scale (each lane is a different row), but we can keep an `<16 x i16>` per-row scale vector and use `pmaddwd` against the `maddubs` output instead of `vpmulld` on the i32 accumulator. | `scale="madd16"` | Replaces a 10-cycle `vpmulld` per block per row-lane with a 5-cycle `vpmaddwd` that also removes the separate ones-vector `pmaddwd`. Modest but cheap. | + +--- + +## 4. IQ2_XS + +Packing: per 32 weights, four `uint16` words, each = 9-bit grid index + 7 sign bits; the 8th +sign is parity. Per-16 scales in a nibble-packed `scales[QK_K/32]`. + +### (a) llama.cpp x86 (AVX2), quants.c:2778-2900 + +1. **Vectorized index extraction.** One 32-byte load of eight `uint16` words; `aux_gindex = + _mm256_and_si256(q2_data, m511)` (x86:2842). The result is written to a stack union and + read back as `uint16 gindex[16]` (x86:2823-2824, commented *"somewhat hacky, but gives a + significant boost in performance"*). +2. **Parity by `pshufb`.** `partial_sign_bits = q2 >> 9`, `partial_sign_bits_upper = q2 >> 13`, + XOR them, then `_mm256_shuffle_epi8(bit_helper, ...)` with a 16-entry nibble table that + returns 0x00/0x80 (x86:2844-2849). That is a 4-bit population-parity computed as a + single `vpshufb`, giving the 8th sign bit in bit 7. No `ksigns` table read at all on this + arch. +3. Grid compose: four `_mm256_set_epi64x` of four `iq2xs_grid[...]` reads each + (x86:2856-2863). +4. **Sign expansion by shuffle + cmpeq.** For each 32-byte group: `vpshufb` broadcasts the + group's sign byte across 8 lanes (`block_sign_shuffle_1/2`, x86:2796-2803), `AND` with + `bit_selector_mask` = `{1,2,4,...,128}` repeated (x86:2804-2807), `cmpeq` against the same + mask -> 0x00/0xFF, `OR` with `mone` -> `+/-1`, then `_mm256_sign_epi8(q8, ...)` + (x86:2871-2885). **Signs on the activation; grid stays unsigned.** Four vector ops per 32 + weights. +5. Scales: one 8-byte `memcpy`, nibble split and interleave, `scales = (stmp<<1) | 1` + (x86:2832-2835). Per-block broadcast via `_mm_shuffle_epi8(scales, get_scale_shuffle(ib32))` + + `cvtepi8_epi16` (x86:2892-2895), then `madd_epi16` (x86:2897-2900). +6. Two i32 accumulators, `fmadd`, `0.125f` at the end (x86:2903-2907). + +An SSE path mirrors all of this at 128 bits (x86:2909-3070). + +### (b) llama.cpp arm64, quants.c:3693-3763 + +1. **No parity arithmetic** - `signs64 + (q2[i] >> 9)` reads `keven_signs_q2xs` directly with + the 7-bit code (arm:3739-3742), and the table already carries the parity completion. The + x86 `pshufb` parity trick exists only because a 128-entry x 8-byte table read does not + vectorize on x86 the way `vld1_s8` does on NEON. +2. Grid: `vcombine_s8(vld1_s8(iq2xs_grid + (q2[0] & 511)), ...)` (arm:3735-3738). +3. Signs on the **weights**, `vmulq_s8` (arm:3743-3746). +4. Four `ggml_vdotq_s32` (arm:3747-3750). +5. **The best scale/reduction shape in the whole ARM file**: + `p = vpaddq_s32(vpaddq_s32(p1,p2), vpaddq_s32(p3,p4))` collapses four per-block i32x4 dot + vectors into one vector whose 4 lanes are the 4 block sums, then a single + `vmlaq_s32(sumi, p, scales32.val[ib64])` applies four per-block scales at once + (arm:3751-3752). Integer accumulation throughout; one `vaddvq_s32` per *superblock* + (arm:3755), not per block. Scales pre-expanded to `int32x4x4_t` outside the k-loop + (arm:3722-3731). + +### (c) What ours does today + +`emit_iq2xs_gather` (gemm_gen.das:1689-1745). Identical skeleton to IQ2_XXS: + +- The `uint16` word is assembled from **two separate `i8` loads plus a shift and an OR** + (gemm_gen.das:1711-1716). The offsets are `lp%4 in {0,2}` within the same 4-byte column, so + a single 2-byte load would do. +- `ksigns` lookup from the 7-bit high field (gemm_gen.das:1718-1720) - the parity path, same + as NEON's, not x86's `pshufb` trick. Correct choice for a scalar decode. +- Grid: two `i32` loads from the `[1024 x i32]` low/high pair global + (gemm_gen.das:1578-1632, 1725-1727). +- Same 3-GPR-op negate and same 4-byte panel store (gemm_gen.das:1728-1732). +- Consumption: `iq2 = true` (gemm_gen.das:1817), so the lo and hi 16-weight halves keep + **split accumulators** `a[]`/`a1[]` and get separate per-16 unsigned scales + (gemm_gen.das:1893-1898, 1912-1919, 1932-1937) - the per-16 granularity IQ2_XS and IQ2_S + need. That part is right and is structurally what ARM's `vpaddq` + `vmlaq_s32` achieves, + without the reduction. + +### (d) Deltas, ranked + +Same 1/2/5/6/7 as IQ2_XXS, plus: + +| # | Delta | Candidate seat | Why | +|---|---|---|---| +| 3' | **One `i16` load for the packed word.** Replace the two `i8` loads + shift + OR (gemm_gen.das:1711-1716) with one aligned 2-byte load. | none - direct fix | 3 uops x 32 x mr per superblock, free. | +| 4' | **Split the sign byte into the plane at repack time.** IQ2_XS is the one format where the parity-completed byte does not fit for free (9 + 8 = 17 bits > 16). Costs +1 bit per 8 weights if adopted; race it against the `ksigns` load. | repack-side | Lower priority than for IQ2_XXS / IQ3_XXS where it is free. | +| 8 | **Race the `pshufb` parity form** as an alternative to the `ksigns` table read, once the sign path is vectorized (delta 2). llama.cpp measured it as *"a significant boost"* (x86:2822) on their layout. | `psign="parity"` | Only interesting if delta 2 lands first; on a scalar-index decode the table read wins. | + +--- + +## 5. IQ2_S + +Packing: 8-bit `qs` index + 2 bits from `qh` = 10-bit index into `iq2s_grid[1024]`; +**explicit sign bits** (`QK_K/8` bytes at `qs + QK_K/8`), no parity trick; nibble-packed +per-16 scales. + +### (a) llama.cpp x86 (AVX2), quants.c:3075-3160 + +1. Grid compose: `_mm256_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32] << 2) & 0x300)], ...)` + (x86:3123-3130) - index arithmetic done **scalar, in the address expression**, four + 64-bit reads per ymm. Note the shift amounts run `8,6,4,2` down the lanes so the same + `qh` byte feeds all four. +2. Sign expansion: `_mm256_set1_epi32(signs[0] | (signs[1] << 16))` broadcasts 4 sign bytes, + `shuffle_epi8(aux256, mask1)` spreads each byte across 8 lanes (`k_mask1`, x86:3090-3092), + `AND` with `k_mask2` = `{1,2,...,128}`x4 (x86:3094-3096), `cmpeq` -> 0x00/0xFF + (x86:3133-3135). +3. Sign application: `_mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1)` (x86:3136) - a + **two-instruction conditional negate under a 0/-1 mask**, cheaper than + `or(mask,1)` + `psign` used by IQ2_XS, and applied **to the activation**. +4. `maddubs` (x86:3146-3147), then scale via + `_mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32))` + `madd_epi16` + (x86:3148-3149). Scales precomputed once per superblock as + `(nibble << 1) | 1`, `cvtepi8_epi16` (x86:3114-3116). +5. Two i32 accumulators, `fmadd`, `0.125f` at the end (x86:3155-3159). + +### (b) llama.cpp arm64, quants.c:3767-3859 + +1. Grid: four `vcombine_s8(vld1_s8(iq2s_grid + (qs[i] | ((qh[..] << k) & 0x300))), ...)` + (arm:3808-3816). +2. Sign expansion: `vdupq_n_u32(signs[0] | (signs[1]<<16))`, two `ggml_vqtbl1q_u8` with + `k_mask1` halves, `vandq_u8` with `k_mask2`, `vceqq_u8` (arm:3820-3825) - the direct + NEON analogue of the x86 shuffle+cmpeq. +3. Sign application: `vmulq_s8(vorrq_u8(vs, m1), q2s)` (arm:3826-3827) - `+/-1` on the + **weights**. +4. Four `ggml_vdotq_s32` (arm:3838-3841). +5. Scale: `sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32] & 0xf))` (arm:3843-3846) - + scalar integer accumulate, **four `vaddvq_s32` cross-lane reductions per 64 weights**. + Weaker than the IQ2_XS `vpaddq`+`vmlaq_s32` form on the same arch, for no obvious reason. + +### (c) What ours does today + +`emit_iq2s_gather` (gemm_gen.das:1518-1575). + +- Index: one `i8` load of `qs`, one `i8` load of the `qh` byte per block, `shl`/`and`/`or` + to build the 10-bit index, then `shl 1` for the low/high word pair + (gemm_gen.das:1543-1550). +- **Sign byte is read straight out of the plane** (gemm_gen.das:1541-1543) - no `ksigns` + indirection, because IQ2_S stores signs explicitly. This is the format where delta 2 + (vector sign expansion) is cheapest to reach: the sign bits are already plane-resident + and, given the `(p/4)*mr*4 + p%4` column layout (gemm_gen.das:1542), **one 32-byte load + covers 8 rows x 4 consecutive sign bytes at stride 4** - exactly the input a + `vpshufb`+`cmpeq` expansion wants. +- Same 3-GPR-op negate, same 4-byte panel store (gemm_gen.das:1557-1562). +- Consumption: the `iq2` split-accumulator per-16 path (gemm_gen.das:1817, 1893-1898). + +### (d) Deltas, ranked + +| # | Delta | Candidate seat | Why | +|---|---|---|---| +| 1 | Delete the panel (as sec.3). | `gather="reg"` | Same as every format. | +| 2* | **Vector sign expansion, promoted.** For IQ2_S specifically the 8 rows' sign bytes are one 32-byte strided column load away (gemm_gen.das:1542); `vpshufb` to select the wanted byte per row, `vpshufb`/`and`/`cmpeq` to expand to 32 mask bytes. ~5 vector ops replace 8 loads + 8 GPR negates. | `psign="expand"` | Highest-value sign delta of the five, because no `ksigns` table read stands in the way. | +| 3 | Adopt the **`xor`+`sub` conditional negate** (x86:3136) in place of `or(mask,1)`+`psign` wherever the mask is already 0/-1. | folded into `psign="expand"` | 2 ops vs 2 ops on x86, but frees `VPSIGNB` port pressure and works at 512 bits where `llvm.x86.avx2.psign.b` does not exist (see sec.9). | +| 5,6,7 | `vpdpbssd` seat, `bias=128` seat, `pmaddwd` scale fold - as sec.3. | | | + +--- + +## 6. IQ3_XXS + +Packing: 8-bit grid index into `iq3xxs_grid[256]` (4 weights each); a per-32 aux word with +four 7-bit sign codes + 4-bit scale, at `qs + QK_K/4`. + +### (a) llama.cpp x86 (AVX2), quants.c:3260-3316 + +1. Grid compose: `_mm256_set_epi32(iq3xxs_grid[q3[7]], ..., iq3xxs_grid[q3[0]])` + (x86:3290-3295) - **eight 32-bit table reads written directly into one ymm**, twice per + loop iteration. This is the exact composition our layout also needs. +2. `memcpy(aux32, gas, 8)` staging (x86:3296). +3. Signs: four `signs64[(aux32[k] >> 7j) & 127]` per ymm (x86:3297-3300), `keven` table, + parity already baked. +4. `_mm256_sign_epi8` on the activation (x86:3301-3302); `maddubs` (x86:3303-3304). +5. Scale `2*ls+1` via `set1_epi16` + `madd_epi16` (x86:3307-3308); two i32 accumulators; + `fmadd`; `*s = 0.25f * hsum_float_8` (x86:3316). + +### (b) llama.cpp arm64, quants.c:3864-3922 + +1. Grid compose: `ggml_vld1q_u32(iq3xxs_grid[q3[0]], ..., q3[3])` (arm:3896-3899) - the macro + (ggml-cpu-impl.h:82-84) is a brace-initialised `uint32x4_t`, i.e. four scalar dword reads + the compiler materialises with `ld1 {v.s}[lane]` or `ins`. Same shape as `set_epi32`. +2. Signs from `keven_signs_q2xs`, `vcombine_s8` of two 8-byte reads (arm:3901-3904). +3. `vmulq_s8` on the **weights** (arm:3905-3908). +4. Chained `ggml_vdotq_s32` x2 per 32 weights (arm:3909-3910). +5. `sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[0] >> 28))` (arm:3911-3912) - again the + per-block cross-lane reduction and **float** accumulate; `*s = 0.5f * sumf` (arm:3916). + +### (c) What ours does today + +`emit_iq3xxs_gather` (gemm_gen.das:1418-1474). Cleanest of our five gathers: + +- **One aligned `i32` load** for the aux word (gemm_gen.das:1438-1439) - the spelling + IQ2_XXS should copy. +- `ksigns` lookup per sub-lane (gemm_gen.das:1442-1444). +- Two `i8` index loads (gemm_gen.das:1445-1450); grid is the **halved** `[256 x i32]` global + (gemm_gen.das:1362-1377, 1455-1457). +- Same 3-GPR-op negate + 4-byte panel store (gemm_gen.das:1458-1462). +- Consumption: `iq2 = false` (gemm_gen.das:1817), so lo and hi share one accumulator and one + per-32 signed scale (gemm_gen.das:1899-1901, 1920-1921) - matching the per-32 granularity. + +### (d) Deltas, ranked + +1. **Delete the panel** (`gather="reg"`) - the single biggest item, and the composition it + needs is literally x86:3290's `set_epi32`, transposed from "8 grid words of one row" to + "one grid word from each of 8 rows". +2. **Vector sign apply** (`psign="mask"` / folded into delta 1) - deletes 24 GPR ops and + `VPABSB` per 32-byte vector. +3. **Bake the parity-completed sign byte at repack time.** Aux is 4x7 sign bits + a 4-bit + scale, and the scale already lives on the scale row - so four full sign bytes fit in the + same 4 bytes, exactly as for IQ2_XXS. Free; deletes the `ksigns` load. +4. `vpdpbssd` seat; `bias=128` seat (needs the `te.bias` clamp lifted, gemm_gen.das:2136); + `pmaddwd` scale fold - as sec.3. + +--- + +## 7. IQ3_S + +Packing: 8-bit `qs` + one bit from `qh` = 9-bit index into `iq3s_grid[512]`; **explicit +sign bytes** (`x[i].signs`); nibble-packed per-32 scales. + +### (a) llama.cpp x86 (AVX2), quants.c:3384-3475 + +1. **Vectorized index compose.** `idx_l = _mm256_cvtepu8_epi16(load 16 qs bytes)`; + `idx.vec[j] = _mm256_and_si256(_mm256_sllv_epi32(set1_epi32(qh[..]), idx_shift), idx_mask)` + with `idx_shift = {1..8}` and `idx_mask = 256`, then `OR` with the widened `qs` + (x86:3430-3438). Eight 9-bit indices built in one ymm with a variable shift. +2. **The index goes through memory, in the safe direction.** `index_t` is a union of + `__m256i vec[2]` and `uint32_t index[16]` (x86:3406-3411); the vector is stored and the + dwords are read back scalar to address the grid (x86:3441-3450). One wide store feeding + 16 narrow, fully-contained loads forwards fine - the opposite of our panel's one wide + load over 8 narrow stores. +3. **An explicit negative result on hardware gather**, x86:3440: + `// At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than + _mm256_set_epi32. Strange.` - with the `vpgatherdd` version left commented out at + x86:3441-3442. Do not spend time on a gather-instruction spelling. +4. Sign expansion: identical to IQ2_S - `set1_epi32(signs[0] | signs[1]<<16)`, + `shuffle_epi8(mask1)`, `and(mask2)`, `cmpeq` (x86:3452-3459). +5. Sign application: `sub(xor(s2, q8), s2)` (x86:3455, 3460) - **on the activation**. +6. `maddubs` (x86:3464-3465); scale `2*ls+1` from the nibble pair via `set1_epi16` + + `madd_epi16` (x86:3466-3469); two i32 accumulators; `fmadd`; **no global factor** - + `*s = hsum_float_8(accumf)` (x86:3475). + +### (b) llama.cpp arm64, quants.c:3926-4032 + +1. Index compose in vector: `vorrq_u16(vmovl_u8(qs_half), vandq_u16(vshlq_u16(vdupq_n_u16(qh), + hshift), m256))` with `k_shift = {8,7,...,1}` (arm:3985-3986, 3992). +2. Same union round trip - `vec_index_t { uint16x8_t vec_index; uint16_t index[8]; }` + (arm:3944-3947), stored then read back scalar for `ggml_vld1q_u32(iq3s_grid[idx.index[0]], + ...)` (arm:3987-3996). +3. Sign expansion via `vqtbl1q_u8` + `vceqq_u8` + `vorrq_u8(..., m1)` -> `+/-1` + (arm:3998-4012); applied to the **weights** with `vmulq_s8` (arm:4004-4005, 4016-4017). +4. **Scales precomputed with a SWAR trick**: `memcpy(scales32, x[i].scales, 4)`, then + `scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101` and the twin for the + low nibbles (arm:3976-3979) - eight `(2s+1)` scale bytes computed with four GPR ops per + superblock. +5. `sumi1 += vaddvq_s32(p1) * scales8[ib32/2]` (arm:4020-4023) - per-block cross-lane + reduction, scalar integer accumulate; `*s = sumf` (arm:4028). + +### (c) What ours does today + +`emit_iq3s_gather` (gemm_gen.das:1300-1359), the arm the followup entry describes. + +- `qh` byte load per block (gemm_gen.das:1319-1321), sign byte load per sub-lane + (gemm_gen.das:1323-1325), `qs` byte load + `shl`/`and`/`or` per grid word + (gemm_gen.das:1327-1333). **All scalar, one byte at a time** - no `sllv`-style vector index + compose. +- Grid: `[512 x i32]` private constant, `align 64` (gemm_gen.das:1260-1272), one `i32` load + per 4 weights (gemm_gen.das:1339-1341). +- Sign: 16-entry nibble->byte-mask table (`dasllama.iq3s.smask`, gemm_gen.das:1274-1293), then + `(gw ^ mw) + (mw & 0x01010101)` (gemm_gen.das:1342-1344). +- 4-byte panel store (gemm_gen.das:1345-1347). +- Consumption: shared per-32 signed scale path (gemm_gen.das:1899-1901, 1920-1921); scale + applied as `vpmulld` on the i32 accumulator; one `sitofp` + FMA per superblock; global + factor pre-baked into the scale row. + +### (d) Deltas, ranked + +| # | Delta | Candidate seat | Why | +|---|---|---|---| +| 1 | **Delete the panel** - compose 8 grid dwords per weight vector with `insertelement`. | `tune_perm(dot="maddubs", width=256, mr=8, gather="reg", requires="avx2")` | The 0.92x tail's stated cause, and the measurement in followup 61 supports it. | +| 2 | **Vector sign expansion off the plane.** Like IQ2_S, the sign bytes are explicit and plane-resident at stride 4 across rows (gemm_gen.das:1324) - one 32-byte column load + `vpshufb`/`and`/`cmpeq` gives the full 32-byte `+/-1`/mask vector. | `psign="expand"` | Deletes the `smask` table read *and* the 3-GPR-op negate *and* `llvm.abs`. | +| 3 | **Vectorized index compose** (`sllv` shape, x86:3430-3438 / arm:3985-3986) feeding a *wide-store / narrow-load* union - the safe memory direction - instead of per-byte `qs` loads. | `idx="vec"` | Only worth racing after delta 1; the win is ~4 GPR ops per grid word. Note llama.cpp deliberately keeps the *grid read* scalar even here. | +| 4 | Do **not** race `vpgatherdd` for the grid read. | - | x86:3440 records it losing to `set_epi32` on Zen 4. | +| 5 | Adopt the ARM SWAR scale expansion (arm:3976-3979) if the scale row is ever computed at runtime. | - | Ours pre-bakes scales in the repack, so this is already better; noted for completeness. | +| 6,7 | `vpdpbssd` seat; `bias=128` seat; `pmaddwd` scale fold - as sec.3. | | | + +--- + +## 8. Verdict on `modules/dasLLAMA/followup_general.md` entry 61 + +Entry 61 says: + +> The stamped gemv gathers each superblock into an alloca panel and then runs the vector dot +> - a store/load round trip per superblock that a single token never amortizes; llama.cpp's +> per-row form (grid words composed straight into vectors, signs applied to the ACTIVATION +> via shuffle+cmpeq/xor-sub, magnitudes kept unsigned for maddubs) edges it 57.0 vs 52.4 +> tg128 on the zen2. + +**Confirmed, for all five formats, not just IQ3_S:** + +- The panel round trip is real and shared. Every one of `emit_iq3s_gather`, + `emit_iq3xxs_gather`, `emit_iq2s_gather`, `emit_iq2xs_gather`, `emit_iq2xxs_gather` ends + each decoded dword with `LLVMBuildStore(b, w, dp)` into `te.iq3s_panel` + (gemm_gen.das:1345-1347, 1460-1462, 1560-1562, 1730-1732, 1792-1794), and + `emit_block_iq4xs` reloads it as `` (gemm_gen.das:1880-1883). The panel is a + GEMV-only cost: in tile mode `te.iq3s_panel` is null (`kq_tile_gen_impl` never sets it, + gemm_gen.das:3200-3244) and the byte-expanded panel is built once per row-group per + token-block by `unpack_kq_panel_grp` (math_gen.das:2091), amortized over up to TB tokens. + That is exactly why prefill wins and decode loses. +- llama.cpp does compose grid words straight into vectors (x86:2691, 2856, 3123, 3290, 3444; + arm:3664, 3735, 3808, 3896, 3987) and never materializes a decoded-weight buffer. +- Magnitudes-stay-unsigned-for-`maddubs` is right on x86 (grid is the u8 operand at + x86:2699, 2887, 3146, 3303, 3464). + +**One clause needs correcting:** + +- *"signs applied to the ACTIVATION"* is **not portable to our layout.** It is true of + llama.cpp's x86 kernels only, and it is a consequence of their one-row-at-a-time shape. + In our `mr`-interleaved GEMV the lanes of one weight vector are 8 different output rows + with 8 different sign patterns, while the activation dword is splatted across all lanes - + so a single `psign` on the activation cannot be correct. llama.cpp's own **arm64** kernels + make the same concession for the same reason (`sdot` is signedxsigned): they apply signs to + the **weights** with `vmulq_s8` at arm:3672, 3743, 3826, 3905, 4004. Adopting + "signs-on-activation" would mean abandoning row interleave (`mr = 1`), which would give + back the per-row horizontal reduction we currently avoid entirely. Not worth it. +- What *is* portable, and is the real content of the delta, is: **keep the grid magnitudes + unsigned and build the sign mask as a vector**, so `psign(x_splat, mask|1)` replaces + `abs(panel_bytes)` + `psign(x_splat, panel_bytes)` and the 3-GPR-op-per-dword negate + disappears with the panel. The `abs`+`psign` pair is dropped outright only by + `vpdpbssd` (signedxsigned, gemm_gen.das:344-347) or by `bias=128` + (gemm_gen.das:339-343) - neither of which is seated for any grid format today. +- Entry 61 is also **under-scoped**: it is filed as an IQ3_S item, but the panel, the gather + shape, the scalar sign negate and the `abs`+`psign` pair are byte-for-byte identical in all + five gathers. One `gather="reg"` spelling in `emit_block_iq4xs` covers all five at once. + +--- + +## 9. Cross-format section - what one spelling would cover + +All five gathers are the same function with a different index/sign preamble. Ranked by +estimated decode-shape payoff: + +### Tier 1 - one change, all five formats + +**`gather="reg"`: compose the weight vector in registers, delete the panel.** + +Where: `emit_block_iq4xs` (gemm_gen.das:1811-1957) and the five gather emitters. Restructure +the gather loop from *row-major* (`for r in mr { for blk, l, h }`, gemm_gen.das:1305-1352) +to *group-major* (`for blk, j, half { for r in mr }`), emitting `insertelement` of each row's +decoded dword into an `` and bitcasting to `te.vwi8` - the same value +`load_vec(te, te.vwi8, wbase, noff)` returns today (gemm_gen.das:1880-1883), produced without +touching memory. + +Why it is the top item: + +- Removes `256 x mr / 4 = 512` scattered 4-byte stores per superblock (mr = 8) and the + 64 wide reloads, plus the `alloca` traffic. +- Removes the **wrong-direction store-forwarding hazard**: the reload at + `(blk*16 + j*4)*mr` (gemm_gen.das:1875) is a 32-byte load spanning 8 distinct 4-byte + stores written `mr*4 = 32` bytes apart (gemm_gen.das:1345). x86 forwards a narrow load out + of a wide store; it does not forward a wide load out of several narrow stores. llama.cpp's + own memory round trips (x86:2823 `aux_gindex`, x86:3406 `index_t`, arm:3944 + `vec_index_t`) are all in the *safe* direction and carry indices, never decoded weights. +- Worse at width 512 today, better after: the `mr = 16` seat writes 1024 stores into a + 4096-byte panel and reloads with 64-byte loads spanning 16 stores each. +- Rough per-32-byte-vector uop budget: ~51 now -> ~20 composed. + +Seats (one per existing perm row, five formats x four rows): + +``` +tune_perm(mr = 8, gather = "reg"), +tune_perm(dot = "maddubs", width = 256, mr = 8, gather = "reg", requires = "avx2"), +tune_perm(dot = "vpdpbusd", width = 256, mr = 8, gather = "reg", requires = "avxvnni|avx512vnni"), +tune_perm(dot = "vpdpbusd", width = 512, mr = 16, gather = "reg", requires = "avx512vnni,avx512bw"), +``` + +(Suffix derivation: `suffix_piece` in llvm_tune.das:1317-1323 renders a string arg as +`name_value`, so these stamp as `..._gather_reg`; add them to each `tune(... fallback = ...)` +chain in priority order.) + +### Tier 2 - one change, all five formats + +**`psign="mask"` / `psign="expand"`: build the sign vector, stop negating in GPRs.** + +Composed with tier 1: keep the raw grid dword as the unsigned magnitude (it already is - +ggml-common.h:561, 1018, 1053), compose a second vector of 0x00/0xFF masks, and apply +`vpor(mask, 1)` + `VPSIGNB` to the splatted activation - or the cheaper +`sub(xor(mask, x), mask)` conditional negate llama.cpp uses at x86:3136/3455. Deletes: + +- the three GPR ops per dword at gemm_gen.das:1342-1344 / 1458-1460 / 1557-1559 / + 1728-1730 / 1790-1792 (~24 ops per 32-byte vector), and +- `abs_w` / `llvm.abs` / `VPABSB` from the inner loop (gemm_gen.das:370-375, 1888-1889). + +Two sub-variants worth racing separately: + +- `psign="mask"` - compose the mask vector from the existing 16-entry `smask` global with + `insertelement` (8 more folded loads). +- `psign="expand"` - read the sign bits vectorized off the plane and expand with + `vpshufb`+`and`+`cmpeq` (llama.cpp's `k_mask1`/`k_mask2`, x86:3090-3096 / arm:3782-3786). + Cheapest for **IQ2_S** and **IQ3_S**, whose sign bytes are plane-resident at stride 4 + across rows (gemm_gen.das:1542, 1324); needs one `ksigns` scalar read per 8 weights for + IQ2_XXS/IQ2_XS/IQ3_XXS unless the repack change below lands. + +Note `llvm.x86.avx2.psign.b` is only wired at `width == 256` (gemm_gen.das:2185-2190); the +512-bit leg already falls back to a `select` (gemm_gen.das:358-362). The `xor`+`sub` form +works at both widths and would simplify that split. + +### Tier 3 - seats only, zero generator change + +**`dot = "vpdpbssd"` for all five formats.** The panel/composed bytes are signed and the +activation is signed, so `dot_lane` takes the native `s8*s8` path with no `abs` and no +`psign` (gemm_gen.das:344-347, 370-372). `perm_declines` already permits it at width 256 +under `avxvnniint8` (gemm_gen.das:212-220). One line per format: + +``` +tune_perm(dot = "vpdpbssd", width = 256, mr = 8, requires = "avxvnniint8"), +``` + +Only helps AVX-VNNI-INT8 hosts (not zen2, not zen4), so file it as cheap coverage, not as +the fix for the measured tail. + +### Tier 4 - repack-side, three of five formats + +**Bake the parity-completed sign byte into the plane.** `ksigns_iq2xs` maps a 7-bit code to +the 8-bit sign byte (ggml-common.h:513). For **IQ2_XXS** and **IQ3_XXS** the aux word is +4x7 sign bits + a 4-bit scale, and the scale nibble already moved to the scale row +(gemm_gen.das:1746-1748) - so four full sign *bytes* occupy the same four bytes, at zero +size cost, and the `ksigns` load disappears (gemm_gen.das:1442-1444, 1775-1776). For +**IQ2_XS** it costs +1 bit per 8 weights (9 index bits + 8 sign bits > 16), so race it. This +also promotes IQ2_XXS/IQ3_XXS into the cheap `psign="expand"` class alongside IQ2_S/IQ3_S. + +### Tier 5 - small, local, do anyway + +- **IQ2_XXS aux word**: four `i8` loads + 3 shifts + 3 ORs (gemm_gen.das:1766-1770) where + IQ3_XXS uses one aligned `i32` load of the identically-aligned field + (gemm_gen.das:1438-1439). 6 uops x 8 blocks x mr rows per superblock, free to reclaim. +- **IQ2_XS packed word**: two `i8` loads + shift + OR (gemm_gen.das:1711-1716) where one + 2-byte load suffices (`lp % 4 in {0,2}`, so the pair never straddles the 4-byte column). +- **Panel/vector load alignment**: `load_vec` hard-codes `align 1` + (gemm_gen.das:376-379) although the alloca is `align 32` (gemm_gen.das:3082) and every + offset is a multiple of `mr*4`. Moot once the panel is gone, but it also affects the tile's + reads of the unpacked panel. +- **`gkstep` is inert for these formats**: `kq_gemv_gen_impl` pins `SliceArgs(kstep = 1)` + (gemm_gen.das:3084) and never reads `p.gkstep`, while `perm_declines` still validates it + (gemm_gen.das:169). Any `gkstep` seat added to a grid format would be a no-op clone. +- **`bias = 128` is clamped off** for every `kq` format at gemm_gen.das:2136 + (`te.bias = (te.mx4 || te.q51 || te.kq != 0) ? 0 : p.bias`). Since we synthesize the panel + bytes ourselves, `w ^ 0x80808080` is a free per-byte `+128` (no carries), so lifting the + clamp for the five panel formats would make the `bias128` lattice - which drops `abs` and + `psign` entirely and folds the `-128*Sumx` correction into the accumulator init + (`bias_acc_init`, gemm_gen.das:400-427) - available on VNNI hosts. **Not** valid under + `maddubs`: `(w+128) x x` pair sums reach 65280 and saturate `i16`, which is why the knob is + documented `vpdpbusd`-only (gemm_gen.das:44-47, 170-171). + +### What we already do better, and should not regress + +- **No horizontal reduction anywhere.** llama.cpp's arm64 kernels pay 4-8 `vaddvq_s32` per + superblock per row (arm:3678, 3752, 3845, 3911, 4021); IQ2_XXS and IQ3_XXS additionally + accumulate in **float** per 32-weight block (arm:3678-3679, 3911-3912). Our lanes are rows, + so the superblock accumulates in `i32` and converts once (gemm_gen.das:1943-1953). +- **Global factor pre-baked.** llama.cpp multiplies by `0.125f` / `0.25f` / `0.5f` per row at + the end (x86:2713, 3161, 3316; arm:3684, 3760, 3916). Ours rides the repacked scale row. +- **Per-16 split accumulators for IQ2_S/IQ2_XS** (gemm_gen.das:1817, 1893-1898) match the + formats' actual scale granularity without any extra reduction - structurally what + llama.cpp's best ARM arm (IQ2_XS `vpaddq`+`vmlaq_s32`, arm:3751-3752) achieves, and better + than what its IQ2_S arm does (scalar `vaddvq` x scalar scale, arm:3843-3846). +- **Q2_K needs no work.** Our `k2` arm decodes 2-bit lanes in-register with a shift and an + `and` (gemm_gen.das:1049-1056) - the same technique as llama.cpp AVX2 (x86:1626-1629) and + NEON (arm:1988-1996) - with no panel, no gather, no table. That is exactly why it measures + at parity, and it is the control that proves the gap in the other five is the *decode*, + not the dot or the fold. diff --git a/plans/kernel_parity_research_kquant.md b/plans/kernel_parity_research_kquant.md new file mode 100644 index 0000000000..a82e0ef17e --- /dev/null +++ b/plans/kernel_parity_research_kquant.md @@ -0,0 +1,445 @@ +# Why our zen2 decode kernel loses on Q3_K / Q5_K / Q6_K and wins on Q4_K + +Read-only study. Reference tree `D:\Work\llama.cpp`, ours `D:\Work\daScript-iquant`. No file in +either tree was modified. + +Measured, zen2 AVX2, 1 thread, m=4096 k=14336, one token (us/call, ours vs llama.cpp): + +| fmt | ours | llama.cpp | ratio | weight+scale bytes / 256-weight row | +|---|---|---|---|---| +| q4_K | 2089 | 2539 | **1.22x ahead** | ours 148, llama 144 | +| q2_K | 2146 | 2121 | 0.99x | ours 84, llama 84 | +| q3_K | 3317 | 2638 | **0.80x** | ours 114, llama 110 | +| q5_K | 3628 | 3286 | **0.91x** | ours 180, llama 176 | +| q6_K | 4203 | 3785 | **0.90x** | ours 210, llama 210 | + +--- + +## 0. Which llama.cpp code path the reference numbers came from + +**The plain `ggml_vec_dot_*_q8_K` in `ggml/src/ggml-cpu/arch/x86/quants.c`, not the repack path.** + +- The repack ("extra buffer type") GEMV only fires when the weight tensor was allocated in + `ggml_backend_cpu_repack_buffer_type()` - `D:\Work\llama.cpp\ggml\src\ggml-cpu\repack.cpp:4774-4781`. +- `test-backend-ops` allocates every tensor with `ggml_backend_alloc_ctx_tensors(ctx, backend)`, i.e. + the backend's *default* buffer type - `D:\Work\llama.cpp\tests\test-backend-ops.cpp:1546`, + `:1555`, `:1805`. So the repack buffer type is never selected and the repack GEMV never runs. + +What repack support even exists on x86 matters for the endgame, though: + +- **q4_K has an AVX2 repack GEMV** - `repack.cpp:4599-4604` selects `q4_K_8x8_q8_K` on + `ggml_cpu_has_avx2()`; the kernel is `ggml_gemv_q4_K_8x8_q8_K`, + `ggml/src/ggml-cpu/arch/x86/repack.cpp:1464`, AVX2-guarded at `:1486`. +- **q2_K's repack is AVX512-only** - `repack.cpp:4627-4632` gates on `ggml_cpu_has_avx512()`; + the kernel `ggml_gemv_q2_K_8x8_q8_K` (`arch/x86/repack.cpp:1713`) is AVX2-guarded at `:1732` + but is never *selected* on a zen2 host. +- **q3_K, q5_K and q6_K have no x86 repack at all** - `repack.cpp:4644-4659` offers q5_K/q6_K + only under `ggml_cpu_has_neon()`; q3_K appears nowhere in the selector. + +So for q3/q5/q6 on zen2, the vec_dot we compared against is the only thing llama.cpp has, and our +comparison is fair. **For q4_K it is not the whole story**: a real llama.cpp run with a loaded model +would take `ggml_gemv_q4_K_8x8_q8_K` instead, which is a row-interleaved 8-column kernel much closer +in shape to ours. Our +22% on q4_K is against llama's *fallback*. + +**AVX512 does not change any of these four kernels.** Each function has exactly one +`#if defined __AVX2__ ... #elif defined __AVX__` pair and no AVX512 arm: q3_K `quants.c:1784`/`:1884`, +q4_K `:2057`/`:2122`, q5_K `:2235`/`:2314`, q6_K `:2439`/`:2510`. The VNNI helper that does exist +(`mul_sum_us8_pairs_float`, `quants.c:105-119`) is used only by the q4_0/q8_0-family dots; none of +the K-quant dots call it. On an AVX512 host llama.cpp runs the identical AVX2 code for these four. + +--- + +## 1. Per-format op budgets, side by side + +### Ground rules for the counts + +- llama.cpp: one 32-byte vector holds **32 consecutive weights of one row**; the activation vector + is a plain 32-byte `q8` load shared by that vector. +- Ours (seat `dot_maddubs_width256_mr8`, from + `D:\Work\daScript-iquant\modules\dasLLAMA\performance\defaults\x86-avx2.tune-defaults.json:35-38`, + giving `width=256`, `mr=8`, `dot=maddubs`; `rv = width/32 = 8`, `rq = mr/rv = 1` - + `dasllama_gemm_gen.das:1883-1884`): one 32-byte vector holds **4 bytes from each of 8 rows**; the + activation is one dword broadcast from memory (`bcast_dword`, `dasllama_gemm_gen.das:844-853`). + So one of our vectors also delivers 32 weight slots - the *unpack cost per vector* is the + apples-to-apples comparison. +- Our unit of emission is a **site** = one `(blk, j)` pair, `blk` in 0..7, `j` in 0..3, 32 sites per + superblock - `dasllama_gemm_gen.das:1002` (`for blk`), `:1027` (`for j`). Each site produces two + 32-byte vectors (`wlo`, `whi`). One superblock-group = 8 rows x 256 weights = 2048 slots. +- I count AVX2-level vector operations after the obvious InstCombine folds (in particular + `shl(and(lshr(X,a),m),b)` collapses to one shift plus one mask; LLVM does this before i8-vector + shift legalization, so our `(x >> a) & m << b` chains realize as 2 ops, not 4). Scalar prologue + arithmetic is not counted on either side. These are static counts, not measured uops. + +### Q4_K - the control we win + +**llama.cpp** (`quants.c:2057-2113`). Prologue: the 12-byte 6-bit scale/min packing is unpacked +with three scalar 32-bit words - `:2069-2074` (`memcpy(utmp,...,12)` then the kmask1/2/3 shuffle, +i.e. `get_scale_min_k4` done four sub-blocks at a time), widened once with `vpmovzxbw` at `:2079`. +The min term is folded **once per superblock**: bsums load `:2081`, `phaddw` `:2082`, +`pmaddwd` against the 8 mins `:2083`, one `fmadd` `:2084` - the whole eight-sub-block min dot is a +single `pmaddwd`, because the 8 mins of one row sit in one 128-bit register. + +Main loop `:2091-2110`, 4 iterations of 64 weights: + +| what | line | ops | +|---|---|---| +| 2 scale shuffles | 2093-2094 | 2 | +| 1 nibble load | 2096 | 1 | +| `q4l = and(bits, 0xF)` | 2097 | 1 | +| `q4h = and(srli(bits,4), 0xF)` | 2098 | 2 | +| 2 activation loads | 2100, 2104 | 2 | +| 2 `vpmaddubsw` | 2101, 2105 | 2 | +| 2 `vpmaddwd` (widen **and** scale in one op) | 2102, 2106 | 2 | +| 2 `vpaddd` | 2107, 2109 | 2 | + +15 per 64 weights x 4 = 60, plus ~15 prologue/epilogue vector ops -> **~75 ops per 256 weights of +one row**. Weight-plane loads issued: 4, distinct: 4 (1.00x). + +**Ours** (`dasllama_gemm_gen.das:1059-1062` for the unpack, `:1081-1108` for the dots): + +| what | line | ops/site | +|---|---|---| +| 1 nibble load (32B) | 1060 | 1 | +| `wlo = and(nv,15)` | 1061 | 1 | +| `whi = lshr(nv,4)` (vpsrlw+vpand on i8) | 1062 | 2 | +| 2 `vpbroadcastd` from memory | 1085-1086 / 844-853 | 2 | +| 2 `vpmaddubsw` + 2 `vpaddw` (i16 chain) | 1097-1101 / 872-878 | 4 | + +10 x 32 sites = 320. Per sub-block: one i16 flush (`vpmaddwd`+`vpaddd`, `:1123-1128`) = 2 x 8 = 16. +Per sub-block fold (`:1154-1160` scale/min loads, `:1181-1189` the integer fold): sc load + `vpmovzxbd` += 2, mn load + `vpmovzxbd` = 2, two scalar bsum loads + add + broadcast = 4, `vpmulld`+`vpaddd` for +`iacc` = 2, same for `bacc` = 2 -> 12 x 8 = 96. Superblock epilogue (`:1232-1250`) ~14. + +Total **~446 per 2048 slots = 55.8 ops per 256 weights of one row**. Weight-plane loads issued: 32, +distinct: 32 (1.00x). + +**Verdict: 55.8 vs 75 - we do 0.74x the work per weight.** Predicted 1.34x, measured 1.22x. + +Where the 19 ops come from: our vector covers 8 rows, so every *per-sub-block* cost (the scale +multiply, the min multiply, the accumulate, the flush) is paid once for 8 rows where llama pays it +once per row. Our fold is 12+2 = 14 ops per sub-block per 8 rows = 1.75 per row; llama's is +2 shuffles + 2 `vpmaddwd` + 2 `vpaddd` per 64 weights = 3 per 32 weights. That is the entire margin. +The *unpack* is a dead heat: 1 load + 3 ALU for two vectors, on both sides. + +### Q5_K - we lose 9% + +**llama.cpp** (`quants.c:2235-2308`). Scale unpack identical to q4_K (`:2252-2257`); min term folded +once per superblock via bsums (`:2261-2265`). **The whole 32-byte `qh` plane is loaded once, at +`:2270`, into one register, and a running `hmask` register walks it** (`:2271`, advanced by +`slli` at `:2287` and `:2292`). + +Main loop `:2278-2305`, 4 iterations of 64 weights: + +| what | line | ops | +|---|---|---| +| 2 scale shuffles | 2279-2280 | 2 | +| 1 nibble load | 2282 | 1 | +| `q5l_0 = and(bits, 0xF)` | 2284 | 1 | +| `q5h_0 = slli(srli(and(hbits,hmask),bit),4)` | 2285 | 3 | +| `q5_0 = q5l_0 + q5h_0` (`vpaddb`) | 2286 | 1 | +| `hmask <<= 1` | 2287 | 1 | +| `q5l_1 = and(srli(bits,4),0xF)` | 2289 | 2 | +| `q5h_1` | 2290 | 3 | +| `q5_1` + `hmask <<= 1` | 2291-2292 | 2 | +| 2 activation loads | 2294-2295 | 2 | +| 2 `vpmaddubsw` | 2297-2298 | 2 | +| 2 `vpmaddwd` (widen + scale) | 2300-2301 | 2 | +| 2 `vpaddd` | 2303 | 2 | + +22 x 4 = 88, plus ~16 prologue/epilogue -> **~104 ops per 256 weights of one row**. Weight-plane +loads issued: 4 nibble + 1 qh = 5, distinct 5 (1.00x). **Zero memory touches for the high bit inside +the loop.** + +**Ours** (`dasllama_gemm_gen.das:1064-1068`). k4's 4 ops for the nibbles, then per site: + +| what | line | ops/site | +|---|---|---| +| load 8 qh bytes (one per row) + broadcast to 4 lanes each | 1066 / `load_row_bytes_x4` 889-896 | 1 load + 2 (vpbroadcastq + vpshufb) | +| `or_bit_x10(wlo, hb, maskLo)` = and, cmpeqb, andn/and, or | 1067 / `or_bit_x10` 926-932 | 4 | +| `or_bit_x10(whi, hb, maskHi)` | 1068 | 4 | + +Site total: 2 loads + 13 ALU + 2 broadcasts + 4 madd/add = 21 -> 672. Flush at `j==1,3` +(`:1112`), one chain only since k5 is not per-16, 2 x 2 x 8 = 32. Fold 96. Epilogue ~14. + +Total **~814 per 2048 slots = 101.8 ops per 256 weights of one row**. Weight-plane loads issued: +32 nibble + 32 qh = 64, distinct 64 (1.00x). + +**Verdict: 101.8 vs 104 - op parity.** Predicted 1.02x, measured 0.91x. The 11% residual is not in +the op count; see section 3. + +Note the shape difference precisely: llama's high-bit deposit is **5 ops and 0 loads per 32 +weights**; ours is **4 ops per 32 slots plus a 3-op memory touch per 64 slots = 5.5 ops and 0.5 +loads per 32 slots**. Almost identical arithmetic, but ours re-enters memory 32 times per superblock +where llama enters it once. The reason is structural: llama's 32 lanes all want *the same* bit index, +so one scalar `bit` serves the whole vector; our 4 lanes of a row want bits t=0,1,2,3, so no uniform +shift exists and the bit must be selected with a per-lane constant mask (`maskLo`/`maskHi`, +`dasllama_gemm_gen.das:953-956`) that has to be re-applied to freshly loaded bytes each site. + +### Q6_K - we lose 10% + +**llama.cpp** (`quants.c:2439-2505`). Scales are native `int8[16]`, loaded raw at `:2455`. The `-32` +offset is removed **once per superblock** with a bsums trick: `q8sclsub = slli(madd(q8sums, +scales_16), 5)` at `:2457`, subtracted from `sumi` at `:2504`. So the dot itself is pure unsigned +`maddubs` on 0..63 values with no per-block correction at all. + +Main loop `:2463-2502`, 2 iterations of 128 weights. **One `qh` load feeds four sub-fields**: + +| what | line | ops | +|---|---|---| +| 2 `ql` loads + 1 `qh` load | 2464-2466 | 3 | +| `q4h_0 = slli(and(H,3),4)` | 2468 | 2 | +| `q4h_1 = slli(and(H,12),2)` | 2469 | 2 | +| `q4h_2 = and(H,48)` | 2470 | 1 | +| `q4h_3 = srli(and(H,0xC0),2)` | 2471 | 2 | +| `q4_0..q4_3 = or(and(ql,15) or and(srli(ql,4),15), q4h_*)` | 2473-2476 | 2+2+3+3 | +| 4 activation loads | 2478-2481 | 4 | +| 4 `vpmaddubsw` | 2483-2486 | 4 | +| 4 xmm scale shuffles | 2488-2491 | 4 | +| 4 `vpmovsxbw` + 4 `vpmaddwd` | 2494-2497 | 8 | +| 4 `vpaddd` | 2499-2500 | 4 | + +44 x 2 = 88, plus ~9 prologue/epilogue -> **~97 ops per 256 weights of one row**. Weight-plane loads +issued: 4 ql + 2 qh = 6, distinct 6 (1.00x). The mask constants are chosen so each 2-bit field is +isolated *and* positioned in one shift - no cleanup mask, because the mask already zeroed the bits +that would bleed across the 16-bit shift lane. + +**Ours** (`dasllama_gemm_gen.das:1069-1079`): + +| what | line | ops/site | +|---|---|---| +| 1 `ql` load + `and 15` + `lshr 4` | 1060-1062 | 1 load + 3 | +| **2 `qh` loads** (columns j and j+4) | 1074-1075 | 2 loads | +| `dlo = and(lshr(qhlo,g2),3)` | 1076 | 2 | +| `dhi = and(lshr(qhhi,g2),3)` | 1077 | 2 | +| `wlo |= dlo<<4`, `whi |= dhi<<4` | 1078-1079 | 4 | +| 2 broadcasts + 2 `vpmaddubsw` + 2 `vpaddw` | 1085-1101 | 6 | + +Site total 3 loads + 15 -> 18 x 32 = 576. Flush at `j==1,3` with **both** chains (k6 is per-16, +`:945`, `:1112-1120`) = 8 x 8 = 64. Fold per sub-block for the k63 arm (`:1139-1143` scale loads, +`:1188-1196` the two-scale integer fold): 2 int8 scale loads + 2 `vpmovsxbd` = 4, 2 scalar bsums + 2 +broadcasts = 4, `ia = sc*a0 + mn*a1` = 3, `ba = sc*bs0 + mn*bs1` = 3, 2 accumulate adds = 2 -> 16 x 8 += 128. Epilogue ~9. + +Total **~777 per 2048 slots = 97.1 ops per 256 weights of one row.** + +**Verdict: 97.1 vs 97 - exact op parity.** Predicted 1.00x, measured 0.90x. + +But: **weight-plane loads issued 96, distinct 48 - a 2.00x redundancy.** The offset at `:1071-1073` +is `hbase = 128*mr + (blk/4)*8*4*mr`, `hloOff = hbase + j*4*mr`, `hhiOff = hbase + (j+4)*4*mr` - it +depends on `blk/4` and `j` only, so **each of the 16 qh columns is re-loaded by all four `blk` values +in its group of four.** llama reads its qh plane exactly once. + +### Q3_K - we lose 20% + +**llama.cpp** (`quants.c:1784-1880`). The 12-byte 6-bit scale packing is decoded with four scalar +32-bit expressions into one xmm at `:1800-1805`, the `-32` bias applied with one `psubb` at `:1806`, +widened at `:1807`, split into two broadcast halves at `:1808-1810`. **The whole 32-byte `hmask` is +loaded once at `:1813`** and never re-read. + +The `-4` offset and the high bit are folded into a single subtraction: `q3h_j` is built with +`andnot(hbits, 1< **~108 ops per 256 weights of one row**. Weight-plane loads issued: +2 qs + 1 hmask = 3, distinct 3 (1.00x). + +**Ours** (`dasllama_gemm_gen.das:1033-1046`). We do **not** pay llama's double-maddubs: we compose an +unsigned 0..7 byte (`wlo = lo2 | (hbit<<2)`, `:1045-1046`) and remove the `-4` once per superblock +through the bsum fold (`koff = 4`, `:946`; applied at `:1238-1240`). One `maddubs` per vector. + +| what | line | ops/site | +|---|---|---| +| **4 loads**: `q3l` col `(blk/4)*8+j`, `q3h` col +4, `h3l` col `j`, `h3h` col `j+4` | 1037-1040 | 4 loads | +| `lo2 = and(lshr(q3l,sh),3)`, `hi2` | 1041-1042 | 4 | +| `lob = ((h3l>>blk)&1)<<2`, `hib` | 1043-1044 | 4 | +| 2 `vpor` | 1045-1046 | 2 | +| 2 broadcasts + 2 `vpmaddubsw` + 2 `vpaddw` | 1085-1101 | 6 | + +Site total 4 loads + 16 -> 20 x 32 = 640. Flush (per-16, two chains, at `j==1,3`) 64. Fold (k63 arm) +128. Epilogue ~9. Total **~841 per 2048 slots = 105 ops per 256 weights of one row.** + +**Verdict: 105 vs 108 - we are marginally *ahead* on op count.** Predicted 1.03x, measured 0.80x. + +But: **weight-plane loads issued 128, distinct 24 - a 5.33x redundancy**, the worst of the four. +Two independent causes, both visible in the offsets: +- `c0 = (blk/4)*8 + j` (`:1035`) -> each of the 16 qs columns is loaded by the 4 `blk` values in its + group (64 issued / 16 distinct). +- `h3l` offset `(64 + j*4)*mr` and `h3h` offset `(64 + (j+4)*4)*mr` (`:1039-1040`) **do not depend on + `blk` at all** -> each of the 8 hmask columns is loaded by all 8 `blk` values (64 issued / 8 + distinct). + +### The two tables together + +Ops per 256 weights of one row, and weight-plane load redundancy per superblock: + +| fmt | llama ops | our ops | our ops ratio | our loads issued/distinct | measured | +|---|---|---|---|---|---| +| q4_K | 75 | 55.8 | 0.74x (better) | 32 / 32 = **1.00x** | **1.22x** | +| q5_K | 104 | 101.8 | 0.98x | 64 / 64 = **1.00x** | 0.91x | +| q6_K | 97 | 97.1 | 1.00x | 96 / 48 = **2.00x** | 0.90x | +| q3_K | 108 | 105 | 0.97x | 128 / 24 = **5.33x** | 0.80x | + +Op count alone predicts +34% / +2% / 0% / +3%. The measured results are +22% / -9% / -10% / -20%. +**The op count explains q4_K and nothing else. The load-redundancy column is what tracks the losses.** + +--- + +## 2. The deltas, ranked by decode payoff + +Each entry says what it is: emitter change (IR only, no plane bytes move), repack-layout change +(plane bytes move -> `IMAGE_VERSION` bump, `dasllama_image.das:41`), or seat/knob. + +### D1 - q3_K: hoist the shared column loads out of the `blk` walk. **Emitter change.** +Payoff: largest single item on the board. 128 issued loads become 32 (the distinct count is 24), and +the CSE'd live set drops from 24 ymm values (16 qs columns + 8 hmask columns = 768 bytes, against 16 +architectural ymm registers) to about 4. + +Shape: keep the existing `(blk, j)` nest but split `blk` into a group of four +(`blk = 4*g + i`, g in 0..1, i in 0..3) and emit the loads at the `(g, j)` level. +For fixed `(g, j)` the four `blk` values share `q3l` (col `8g+j`), `q3h` (col `8g+j+4`), `h3l` +(col `j`) and `h3h` (col `j+4`) - see the offsets at `dasllama_gemm_gen.das:1037-1040` - and differ +only in the constant shifts `sh = 2*i` and `blk`. Live set inside the group: 4 loaded column vectors ++ 4 sub-blocks x 2 halves = 8 i16 chains, which fits. +No plane byte moves. No `IMAGE_VERSION` bump. No Vulkan/Metal impact (see D6 note). +Estimated: -96 loads/group (~-11% static ops) plus whatever spill traffic disappears; on the measured +0.80x this is the difference between "worst format" and "roughly at parity". + +### D2 - q6_K: the same hoist for the `qh` columns. **Emitter change.** +`hloOff`/`hhiOff` (`dasllama_gemm_gen.das:1072-1073`) depend on `blk/4` and `j`, not `blk`. Emitting +the two `qh` loads once per `(blk/4, j)` and looping the four `blk` values inside turns 64 qh loads +into 16. `ql` loads stay per-site (they are already 1.00x). Live set: 2 qh + 4 ql + 8 chains. +Estimated -48 loads/group (~-6% static ops) plus spill relief. + +### D3 - q3_K and q2_K: the i16 chain flushes twice as often as the bounds require. **Emitter change.** +`dasllama_gemm_gen.das:1112` flushes at `j==1` and `j==3` for every non-k4 format. The stated bound +(`madd16_acc`, `:869-871`) is one `vpmaddubsw` pair-sum <= `2*qmax*127`: +- k6 `2*63*127 = 16002` -> 2 madds = 32004, at the i16 limit. Correct as written. +- k5 `2*31*127 = 7874`, one chain carrying both halves -> 4 madds = 31496, at the limit. Correct. +- **k3 `2*7*127 = 1778`** -> 4 madds = 7112. A single flush per sub-block is safe (18 would be). +- **k2 `2*3*127 = 762`** -> 4 madds = 3048. Same. +The code comment at `:1110-1111` already says "k3, conservatively". Fix: make the flush predicate +`(j == 3)` for k3/k2. -4 ops per sub-block = -32/group, about -4% on k3 and -5% on k2. + +### D4 - q5_K: no clean lever exists; say so and stop looking. +Our k5 site is 2 loads + 13 ALU (`:1064-1068` plus the k4 nibble ops), llama's equivalent is +1 load-free deposit of 5 ops per 32 weights. Per weight the two are within 5% of each other and the +total op counts are 101.8 vs 104. The three candidate rewrites all fail: +- Rewriting `or_bit_x10` (`:926-932`) from `icmp-ne + select` to `vpand(vpcmpeqb(and,mask),16)` is the + same 4 ops. No win. +- **Repack-layout change**: pre-expanding the qh plane from 1 byte per (row, site) to 4 (so the bit is + already lane-local and the broadcast+shuffle at `:1066` disappears) saves about 4 ALU/site = + -128 ops/group (-16%), but grows the k5 weight plane from 160 to 256 bytes per superblock (+60%). + Our k5 currently moves 4096 x 56 x 180 B = 41.3 MB per call in 3628 us = 11.4 GB/s, against the ~16.3 GB/s our k4 demonstrates + is reachable on this core; the widened plane would be 58.7 MB, and even at 16.3 GB/s that is 3600 us -- no better than today. **Reject.** +- Splitting the dot into a nibble part plus a `16 *` high-bit part (llama's q3 trick applied to q5) + needs the same 4-op predicate to build the 0/1 vector, and adds a `maddubs`. **Reject.** +The honest read: q5_K's 9% is scheduling, not arithmetic - the per-site chain +`load(8B) -> vpbroadcastq -> vpshufb -> vpand -> vpcmpeqb -> vpandn -> vpor -> vpmaddubsw` is 8 deep +and repeats 32 times in one straight-line block. D5 is the lever that addresses it. + +### D5 - a sub-block unroll knob for the kq GEMV. **Emitter change + schema knob.** +`emit_block_kqv2` emits **all 32 sites of a superblock as straight-line IR** - `dasllama_gemm_gen.das:1002` +(`for blk in range(8)`) and `:1027` (`for j in range(4)`) are das-level compile-time loops, and the kq +GEMV driver hardcodes one superblock per emitted iteration (`SliceArgs(..., kstep = 1)`, +`dasllama_gemm_gen.das:2830`). That is 600-900 IR instructions in one basic block with 16 ymm +registers available. The generic `gkstep` knob documented at `:29-31` is not reachable from the kq +GEMV path. Adding an `sbstep` (sub-blocks per emitted loop iteration; 8 = today's behaviour, 2 and 4 +available) would give the tuner a handle on the live-set for every kq format at once, and is the only +proposal here that also helps q5_K. Costs a schema/perm-vector field; no plane bytes move. + +### D6 - q4_K scale-plane padding: identified, and not worth taking. +Our k4/k5 scale plane is 20 bytes per superblock (`K4_SSB`/`K5_SSB`, `dasllama_kqformat.das:94-96`): +2 d + 2 dmin + 8 decoded `sc` + 8 decoded `mn`, written by +`repack_k4_grp` (`dasllama_repack.das:218-224`) and `repack_k5_grp` (`:270-276`) - the 6-bit packing +is decoded once at load via `k4_sc_mn` (`dasllama_math_default.das:381-391`). llama keeps the packed +12 bytes and decodes in-register (`quants.c:2069-2074`), so its row is 16 bytes. That is +4 bytes per +superblock = **+2.8% weight-plane bytes**, and q4_K is the one format where we are bandwidth-bound +(4096 x 56 x 148 B = 34.0 MB per call in 2089 us = 16.3 GB/s, the highest rate any of the five reaches on this core). Reverting to +a packed plane would cost about 8 ALU ops per superblock-group against 32 bytes saved - roughly +break-even at our measured 2.65 bytes per op - for a repack-layout change and an `IMAGE_VERSION` +bump. Not worth it. The same reasoning says **keep** the k3 pre-decode: `K3_SSB = 18` +(`dasllama_kqformat.das:106`, "6-bit scale - 32, decoded at transcode") costs 4 bytes over llama's 14 +but buys us zero runtime scale decode, and k3 is compute-bound, not bandwidth-bound. + +### D7 - the seat/knob lane is empty for these formats. +`mr` must be a multiple of `rv = width/32 = 8` (`setup_tile_emit`, `dasllama_gemm_gen.das:1883-1884`), +so on AVX2 the only legal values are 8 and 16; 16 doubles the register pressure without changing any +redundancy factor (a wider column is still re-read once per `blk`). `kstep` is pinned to 1 for kq +(`:2830`). `nrsplit` is irrelevant at tokCount = 1. Zen2 has no VNNI, so `dot` is fixed at `maddubs`. +**No retune can move these numbers; every item above is code.** + +### Repack-layout / GPU-reader note +None of D1, D2, D3, D5 moves a plane byte, so none needs an `IMAGE_VERSION` bump. If a future item +does move the grp bytes, the blast radius is CPU-only: the grp repack runs at load and only when the +active backend asks for it (`dasllama_math.das:1182` `g_active_needs_repack`, recorded per model as +`kq_repacked`, `dasllama_load.das:2610`, `:2635`), while the GPU-resident tier reads the plane at the +**disk** stride (`t.k5q[sb0 * K5_QSB]`, `dasllama_gpu_resident.das:267-289`), not the grp layout. A +baked image does carry the repacked planes, so the `IMAGE_VERSION` bump is still required - but the +Vulkan and Metal readers are not on the hook. (Worth a one-line confirmation before anyone relies on +it.) + +--- + +## 3. Why q4_K wins and q5_K/q6_K lose in the same emitter - the hmask/qh suspect is CONFIRMED + +The suspect holds, and the correlation is perfect across all five formats: + +| fmt | high-bit plane? | our loss/win | +|---|---|---| +| q4_K | none | **+22%** | +| q2_K | none | -1% | +| q5_K | `qh`, 1 bit/weight | -9% | +| q6_K | `qh`, 2 bits/weight | -10% | +| q3_K | `hmask`, 1 bit/weight | -20% | + +**Every format with a separate high-bit plane loses; neither format without one does.** The mechanism +is not the arithmetic - section 1 shows we are at op parity or better on all three losers. It is +plane residency: + +- In llama.cpp's row-major layout, a 32-byte register holds the high bits for **32 consecutive + weights of one row**. Q3_K's entire 32-byte `hmask` covers the whole 256-weight superblock and is + loaded once (`quants.c:1813`). Q5_K's entire 32-byte `qh` likewise (`:2270`). Q6_K's `qh` is 64 + bytes, so it is loaded twice, one register per 128-weight iteration (`:2466`). **The high-bit plane + is register-resident for the whole loop, and extraction is pure ALU with a uniform shift, because + all 32 lanes want the same bit index.** +- In our `grp` layout a 32-byte register holds 4 bytes from each of 8 rows, so the high-bit plane + for the same amount of work is **mr = 8 times larger**: q5 256 bytes, q3 256 bytes (32 hmask x 8 + rows), q6 512 bytes - 8 to 16 ymm registers' worth, against 16 architectural registers on AVX2. + **It cannot be register-resident.** Every extraction therefore re-enters memory + (`dasllama_gemm_gen.das:1039-1040` for k3, `:1066` for k5, `:1074-1075` for k6), and because the + four lanes of a row want four different bit positions, the extraction needs a per-lane constant mask + (`maskLo`/`maskHi`, `:953-956`) rather than a uniform shift. +- Q4_K and Q2_K have no high-bit plane at all: their entire weight is inside the nibble/crumb byte + (`dasllama_gemm_gen.das:1060-1062` for k4, `:1051-1054` for k2). There is nothing to re-touch, our + loads are 1.00x, and the 8-row amortization of the fold shows up undiluted - hence +22% on k4. + +Q2_K's near-parity rather than a q4-sized win is the secondary effect and is also visible in the +code: k2 is a per-16-scale format (`per16 = k63 || k2`, `:945`), so it runs split lo/hi accumulators, +two i16 chains, two flushes per sub-block, and four `vpmulld` per sub-block in the fold +(`:1200-1207`) where k4 uses two - plus its plane is only 84 bytes per superblock, giving it about +1.08 bytes per vector op against k4's 2.65, i.e. it is compute-bound where k4 is bandwidth-bound. + +**Ranking of the three losers is set by the load-redundancy factor, not by the plane's bit width.** +q6_K carries twice as many high bits as q3_K but loses less, because its qh columns are re-read 4x +while q3_K's are re-read a weighted 5.33x overall (qs 4x *and* hmask 8x). That is why D1 (q3_K) is +the top item and D2 (q6_K) the second, and why both are pure loop-restructuring in the emitter with +no layout consequence. + +--- + +## 4. Two things to keep in view + +1. **The q4_K win is measured against llama.cpp's fallback.** `ggml_gemv_q4_K_8x8_q8_K` + (`arch/x86/repack.cpp:1464`, selected on AVX2 at `repack.cpp:4599-4604`) is what a real llama.cpp + model run uses on zen2, and it is an 8-column row-interleaved kernel structurally similar to ours. + Any claim of "1.22x ahead on q4_K" should be re-measured against that kernel before it leaves the + building. q3/q5/q6 have no such caveat - llama.cpp has no x86 repack for them. +2. **The prefill tiles are 1.6-2.6x ahead on the same formats**, which is consistent with everything + above: at tokCount = 4 the per-site unpack (including the redundant high-plane loads) is amortized + over four activation dwords instead of one, so the fraction of the loop spent on plane touching + drops by roughly 4x and the losers stop being losers. The decode kernel is the only place where + the unpack cost is paid at full weight. diff --git a/plans/kernel_parity_research_vk.md b/plans/kernel_parity_research_vk.md new file mode 100644 index 0000000000..53a0442fa3 --- /dev/null +++ b/plans/kernel_parity_research_vk.md @@ -0,0 +1,470 @@ +# Technique memo: llama.cpp's cooperative-matrix-2 prefill GEMM vs ours + +Read-only research. Trees: `D:\Work\llama.cpp` at `6c84c7d5d` (2026-08-27) and +`D:\Work\daScript-iquant`. Nothing in either tree was modified. + +Target box for every number below: RTX 5060 Ti (Blackwell GB206, 36 SMs), the box +`D:\Work\daScript-iquant\modules\dasLLAMA\ARCHITECTURE_GPU_VULKAN.md:49` names. + +One environment fact that shapes the whole comparison: the local llama.cpp Vulkan build +reports `GL_NV_cooperative_matrix_decode_vector not supported by glslc` +(`C:\Users\Boris\.claude\jobs\7f9893d9\tmp\tbo_build_vk.log`). llama.cpp therefore runs the +scalar (one element per callback) decode path here, the same as ours. The comparison is +apples to apples today. It stops being so the moment that glslc is upgraded - see delta 7. + +--- + +## 1. llama.cpp's cm2 GEMM, in ten facts + +**F1 - One shader, six pipelines per quant type, three tile geometries.** +Geometry arrives as spec constants `{BLOCK_SIZE, BM, BN, BK, enable_smaller_matrices}` +(`D:\Work\llama.cpp\ggml\src\ggml-vulkan\vulkan-shaders\mul_mm_cm2.comp:31-39`). On coopmat2 the +values are large `{256, 128, 256, 64, 1}`, medium `{256, 128, 128, 64, 1}`, small +`{256, 32, 64, 128, 0}` - and they are byte-identical for legacy quants and for K-quants +(`D:\Work\llama.cpp\ggml\src\ggml-vulkan\ggml-vulkan.cpp:4263-4277`). Each is stamped twice, +unaligned and aligned, giving six (`ggml-vulkan.cpp:4664-4671`). There is **no per-format tile +tuning** on the cm2 path. + +**F2 - Operands: f16 times f16 into an f16 accumulator, stored as f32.** +`mat_a` is `BM x BK` of the quantized weight, `mat_b` is `BK x BN` of the activation, both +`MAT_TYPE = FLOAT_TYPE = float16_t` on coopmat2 (`mul_mm_cm2.comp:98-102`, +`vulkan-shaders-gen.cpp:492-529`). `coopMatMulAdd` accumulates into `ACC_TYPE`, which is +`float16_t` at `GGML_PREC_DEFAULT` and `float` only at `GGML_PREC_F32` +(`vulkan-shaders-gen.cpp:471-475`; the pick is `ggml-vulkan.cpp:7818-7820`). The device is only +admitted to the cm2 path if it advertises fp16 A and B with both fp16 and fp32 accumulators at +workgroup scope and flexible dimensions (`ggml-vulkan.cpp:6806-6838, 6863-6869`). Result is +converted to `D_TYPE = float` for the store (`mul_mm_cm2.comp:493-495`). + +**F3 - There is no integer activation path on coopmat2.** +An f32 `src1` on a coopmat2 device is unconditionally marked non-contiguous +(`ggml-vulkan.cpp:9276-9278`), which forces `quantize_y` false (`ggml-vulkan.cpp:9285`), and the +q8_1 integer-dot `mul_mmq.comp` shader is never even compiled for coopmat2 +(`vulkan-shaders-gen.cpp:626`). Activations are always f16 here. + +**F4 - The f32-to-f16 activation conversion is one dispatch, cached across sibling matmuls.** +It runs through `ggml_vk_cpy_to_contiguous` into the `prealloc_y` scratch, but only when the +last conversion did not already produce the same tensor with the same pipeline +(`ggml-vulkan.cpp:9441-9453`, guarded by `prealloc_y_last_pipeline_used` / +`prealloc_y_last_tensor_used`). Q, K and V therefore pay one conversion between them. + +**F5 - Weights are decoded inside the driver's block load; nothing is pre-dequantized.** +`coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(...), dequantFuncA)` +(`mul_mm_cm2.comp:381, 468`). The tensor layout carries the quant block size +(`setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K)`, `mul_mm_cm2.comp:315-318`) so the +driver resolves the block pointer and hands the callback a `buffer_reference` to it. The weight +stays in ggml's on-disk block layout: no repack, no staging pass, no extra VRAM. + +**F6 - Almost nothing is staged in shared memory, and the one exception is Q4_K/Q5_K scales.** +`mul_mm_cm2.comp` declares only `row_ids` and `ballots_sh`, both under `MUL_MAT_ID` +(`mul_mm_cm2.comp:108, 116`) - contrast the non-cm2 shader, which stages both operands +(`mul_mm.comp:130-131`). The exceptions are the iq codebooks +(`types.glsl:918, 1031, 1201, 1501, 1573, 1683, 1813`, copied by `init_iq_shmem`) and a +**Q4_K/Q5_K-only scale hoist**: `fetch_scalesQ4_K` reads the whole 16-byte scale head of a +superblock as ONE `uvec4` (`dequant_funcs_cm2.glsl:449, 461`, via the +`buffer_reference_align = 16` `block_q4_K_packed128` alias at `:429-431`), and +`store_scalesQ4_K` unpacks it into 8 ready-multiplied `(d, m)` pairs in +`shAscales[8 * (BM+2)]` (`:442-444, 484-523`). Every element decode then reads one `vec2` with +no shifting at all (`:537-540`). The comment states the intent outright: "we decode a tile's +worth of scales into shared memory and then process the whole tile using those scales" +(`:435-440`). + +**F7 - Two tensor loads, one multiply-add, zero barriers per k-step.** +The inner body is exactly `coopMatLoadTensorNV(mat_a, ...)`, `coopMatLoadTensorNV(mat_b, ...)`, +`sum = coopMatMulAdd(mat_a, mat_b, sum)` (`mul_mm_cm2.comp:464-473`). The only barriers in the +k loop are the two inside `store_scalesQ4_K` (`dequant_funcs_cm2.glsl:486, 522`), which fire once +per `QUANT_K = 256` elements of k - that is once per four `BK = 64` steps - and only for +Q4_K/Q5_K. Every other format runs the whole k loop barrier-free. The scale fetch and store are +explicitly pipelined: the next block's scales are fetched while the current block's are being +consumed (`mul_mm_cm2.comp:363-374`). + +**F8 - A clamp-free fast path, entered by construction rather than by luck.** +The fast path requires a whole M tile, a whole N tile against `padded_N`, `start_k % 256 == 0`, +`end_k % BK == 0` and `stride_b % 8 == 0` (`mul_mm_cm2.comp:344-349`). Strides are then masked +`&= ~7` purely to make 16-byte alignment provable to the driver's address analysis +(`:299-306, 350-355`). Crucially, the N side is **padded in the buffer**: +`padded_n = ROUNDUP_POW2(ne11, pipeline->wg_denoms[1])` (`ggml-vulkan.cpp:9319`), so even the last +token tile takes the unclamped path. The unaligned fallback uses clamp-Constant layouts and is +marked `[[dont_unroll]]` (`mul_mm_cm2.comp:607`). + +**F9 - The tail N tile gets a narrower accumulator.** +`enable_smaller_matrices` makes the final column tile allocate `BN/2` or `BN/4` accumulators and +loop over that width (`mul_mm_cm2.comp:36-38, 367-409, 410-452`), so a partial token column does +not carry a full-width accumulator through the whole k loop. + +**F10 - Split-k with a separate vectorised reduce.** +Split fires when `k >= 2048` and the tile grid fills less than half the SMs; the factor is +`shader_core_count / tiles`, capped at 8, with the chunk rounded to 256 and splits shed if the +last chunk would be empty (`ggml-vulkan.cpp:8795-8826`). Partials land in `prealloc_split_k` and a +256-thread `vec4` shader sums them (`ggml-vulkan.cpp:8909-8925`, +`mul_mat_split_k_reduce.comp:17-47`). + +**Worked pick for a 1B prefill shape on this box** (M = 2048 weight rows, N = 512 tokens, +K = 2048, 36 SMs): `tiles_m = 16 * 4 = 64 > 36`, so `prefer_large` is true and the aligned large +pipeline is chosen - BM 128, BN 256, BK 64, 256 invocations +(`ggml-vulkan.cpp:8839-8863`, `9309-9312`). Split-k: 32 tiles, neither `<= 18` nor `<= 24`, so +`split_k = 1` (`ggml-vulkan.cpp:8795-8807`). Grid is 16 by 2 by 1 workgroups +(`ggml-vulkan.cpp:8896`, `8221-8229`). + +Batch and MoE, for completeness: plain batches ride `gl_WorkGroupID.z` with `broadcast2`/ +`broadcast3` (`mul_mm_cm2.comp:253-261`); MoE builds a shared `row_ids` table with a +subgroup-ballot prefix scan and stores through a per-element callback +(`mul_mm_cm2.comp:163-227, 150-161`). + +--- + +## 2. Ours, in the same terms + +**O1 - Same three tile geometries, same workgroup size, same accumulator type.** +`KqCm2BatchT::run` is `[spirv_kernel(local_size_x = 256)]` +(`D:\Work\daScript-iquant\modules\dasLLAMA\dasllama\dasllama_vulkan_classes.das:4825`). A is +`coopmatWgA_f16_128x64` (`:4888`), so BM 128 and BK 64 are fixed; BN is a template constant of +256, 128 or 32 with matching B and accumulator types (`:4809`, `:5336-5361` for q8 and the same +four typedefs repeated per format through `:5713`). The accumulator is +`coopmatWgAcc_f16_128xBN` - f16, like llama.cpp's default - clamped to the f16 range and widened +to f32 only for the store (`:4923-4926`). + +**O2 - Same tile pick, arrived at independently and landing in the same place.** +`cm2_tile_cols` compares occupied against allocated wave slots cross-multiplied, ties to large, +with `cnt <= 64` forcing small and `cnt <= 128` forcing medium +(`D:\Work\daScript-iquant\modules\dasLLAMA\dasllama\dasllama_vulkan_common.das:129-151`). For +d = 2048, cnt = 512, 36 cores it returns 256, the same large tile llama.cpp picks. + +**O3 - Same split-k rule, near-identical constants.** +`cm2_split_k` gates on `n >= 2048`, `d >= 128`, `cnt >= tc`, computes `cores / tiles` capped at +8, rounds the chunk to 256 and sheds stranded splits +(`dasllama_vulkan_common.das:155-179`). The in-kernel plane decomposition is +`dasllama_vulkan_classes.das:4832-4846`; the reduce is a separate class +(`:5722-5751`), encoded at `dasllama_vulkan_prefill.das:355-366`. + +**O4 - Same decode-in-load mechanism.** +`coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode)` +(`dasllama_vulkan_classes.das:4911`), with the block size on the layout +(`tensorLayoutSetBlockSize(fla, 1u, BLKW)`, `:4896`). Weights are never materialised in f16. + +**O5 - Same fast/edge split and the same stride-masking trick.** +Fast path when `m0 + 128 <= pa.d`, the token column is whole or the small tile is stamped, and +`pa.n % 64 == 0` (`:4893`); strides masked `& ~7u` on the B and output layouts (`:4902, 4906`). +Edge path uses clamp-Constant layouts (`:4950-5005`). + +**O6 - Same k-loop shape, one degree more unrolling, zero barriers.** +Two loads and one `coopmatMulAdd` per 64-wide k step, the outer loop unrolled 8 deep for 512 k +per iteration with a scalar drain (`:4907-4922`). The kernel contains exactly one `barrier()`, +before the loop, to publish `wg_blk0` and any staged codebook (`:4887`). llama.cpp unrolls 4 deep +at `QUANT_K = 256` (`mul_mm_cm2.comp:341-342`). + +**O7 - The activation f16 conversion is likewise one dispatch per sibling group.** +`f16cvt_cls` feeds q/k/v (`dasllama_vulkan_prefill.das:518-524`) and gate/up (`:562-568`); +`actf16_cls` fuses the SwiGLU epilogue with the conversion feeding down (`:569-576`). All of it +is recorded into the measured command buffer. This is at least as good as llama.cpp's caching, +and the fused activation is strictly better. + +**O8 - Difference: the weight scale plane is a second SSBO, addressed by hand, per element.** +`wq` (quants) is binding 0 and `ws` (scales) is binding 1 +(`dasllama_vulkan_classes.das:4804-4805`). Every decode callback computes its own scale address +`(wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u` and then extracts the sub-block scale with a 32-bit +load and a **runtime-variable shift** - per decoded element. Q4_K is the worst case, three +separate `ws` dwords at a 20-byte stride plus three variable shifts +(`:5034-5038`). Q6_K (`:5058-5062`), Q3_K (`:5143-5147`), Q2_K (`:5164-5167`), IQ4_XS +(`:5122-5125`), IQ3_S (`:5190-5193`), IQ3_XXS (`:5223-5226`), IQ2_S (`:5269-5273`) and IQ2_XS +(`:5295-5299`) all carry the same shape. llama.cpp reads every scale from the same block the +driver already resolved, and for Q4_K/Q5_K does the extraction eight times per 256 elements +instead of 256 times (F6). + +**O9 - Difference: no shared-memory scale hoist for any format.** +The only workgroup memory declared is `wg_blk0` (4 bytes) plus one gated iq codebook of 32 bytes +to 8 KB (`dasllama_vulkan_classes.das:4808-4823`). There is no `shAscales` equivalent. + +**O10 - Difference: the codebooks are computed per workgroup, not copied.** +`iq3s_gridc[gi] = iq3s_grid_word(int(gi))` and its siblings synthesise the table with arithmetic +at kernel entry (`:4861-4886`), where llama.cpp copies from a constant array +(`types.glsl:1204-1213`). + +**O11 - Difference: a per-workgroup prologue that llama.cpp does not have.** +Each workgroup reads a region record from a `sched` SSBO - five scalar loads +(`dasllama_vulkan_classes.das:3972-3976`) - broadcasts `wg_blk0` through workgroup memory +(`:4849-4851`) and barriers (`:4887`). llama.cpp derives everything from `gl_WorkGroupID` with no +loads and no leading barrier. Ours buys MoE region lists with this; llama.cpp pays for the same +capability with a separate `MUL_MAT_ID` shader. + +**O12 - Difference: no N padding and no narrow-tail accumulator.** +The activation plane is sized to `cnt`, not to a tile multiple, for the large and medium tiles +(`:4901`), so a partial last token column drops to the fully clamped edge path (`:4893`). Only +the small tile rounds its dimension up to 32 and keeps the unclamped load. There is no +`enable_smaller_matrices` analogue. At exactly 512 tokens with BN 256 this costs nothing; at +500 tokens it costs the last tile. + +**O13 - Grid: one-dimensional.** +`vkCmdDispatch(raw, groups, 1u, 1u)` (`dasllama_vulkan_common.das:2469-2478`) with +`groups = ceil(rows/BN) * ceil(d/128) * nsplit` (`dasllama_vulkan_prefill.das:1089-1106`). Push +constants are 16 bytes, `{n, d, map_off, ksplit}` (`dasllama_vulkan_classes.das:2326-2331`); +everything else rides the SSBO region record. llama.cpp pushes a 17-word block with the full +geometry (`mul_mm_cm2.comp:41-70`). + +**O14 - The measured ratios are NOT a uniform band, and the followup says otherwise.** +`followup_vulkan.md:540-546` (entry 34) states "~0.67-0.70x" and "every sb-format cm2 tile lands +in the same band" and "the gap is the shared batch-GEMM tier, not any one format's decode". +The per-format tables in +`D:\Work\daScript-iquant\modules\dasLLAMA\HOW_TO_ADD_A_FORMAT.md`, all 5060 Ti against llama.cpp +b10660 Vulkan at `-ngl 99` on 1B-class vehicles, say: + +| format | pp512 ours / theirs | ratio | cite | +|---|---|---|---| +| IQ4_XS | 15334 / 17060 | 0.90x | `HOW_TO_ADD_A_FORMAT.md:1132` | +| Q2_K | 14544.0 / 16752.7 | 0.87x | `:774` | +| Q3_K | 14031 / 17509 | 0.80x | `:1088` | +| IQ4_NL | 15027.4 / 19177.8 | 0.78x | `:831` | +| IQ2_XXS | - | 0.78x | `:554` | +| IQ2_XS | 14023.3 / 18320.0 | 0.77x | `:606` | +| IQ3_S | 12539.6 / 17865 | 0.70x | `:1020` | +| IQ2_S | 12099.3 / 17377.5 | 0.70x | `:702` | +| IQ3_XXS | 12225.7 / 17807.7 | 0.69x | `:918` | +| Q4_K (the control) | - | 0.67x | `:987` | + +That is a 1.34x spread, not a band. Entry 34 was written on 2026-08-30 +(`followup_vulkan.md:543`) when only k4 and iq3s had Vulkan rows; six formats landed after it +and none updated it. The document contradicts itself at `HOW_TO_ADD_A_FORMAT.md:606-607` +("0.77x - above the 0.70x tier class") and `:753-754` ("0.87x - above the 0.69-0.78 tier +class"). + +**O15 - The "shared tier" diagnosis is refuted by our own Q8 rows.** +Q8_0 on the same tile, the same chain, the same box measures 3B pp 7737.2 = 100.6% of the +reference exe and tinyllama pp 20986 = about 103.5% (`followup_vulkan.md:211-213`). If the deficit +were the shared batch-GEMM tier, Q8 would show it. And the kernel-rate probes already point at +the decode: Q8 cm2 clocks 62.2/64.7 TF/s on gate/down (`followup_vulkan.md:176-178`) while Q4_K +clocks 35.8-38.2 TF/s, "about 70% of Q8-cm2's rate - the nibble+scale extraction" +(`followup_vulkan.md:223-224`); Q6_K sat at 9.3-13.4 TF/s until one byte4 dynamic select was +respelled, after which it hit 32.9 TF/s (`followup_vulkan.md:229-234`). + +The honest caveat is that these Q8 rows are 3B-shape and the per-format rows are 1B-shape, so +shape and decode are confounded in the evidence that exists. Nobody has run them apart. + +--- + +## 3. The deltas, ranked by estimated pp512 payoff on a 1B/3B shape + +Shape assumed throughout: dim 2048-3072, hidden 8192, 512 tokens, so the large tile +(BM 128, BN 256, BK 64) and split-k off on the wide GEMMs, split-k 2 on the narrow k/v +projections. + +### Delta 1 - Hoist the per-(row, sub-block) scale into workgroup memory, llama.cpp's `shAscales` form. Estimated payoff: 10-20% on Q4_K/Q5_K, less on the rest. + +Ours re-derives the scale for every decoded element: a 32-bit load from a second SSBO plus a +runtime-variable shift plus a mask plus a float convert, three times over for Q4_K +(`dasllama_vulkan_classes.das:5034-5038`). llama.cpp does that arithmetic once per (row, +sub-block) - 8 times per 256 elements per row - into `shAscales`, and the element decode is then +a single `vec2` read with no shifting (`dequant_funcs_cm2.glsl:484-523, 537-540`). That is a +32-fold reduction in scale-extraction ALU on the format that is currently our worst. + +This is the strongest hypothesis because Q4_K is simultaneously (a) our worst ratio at 0.67x +and (b) the only format llama.cpp gives a special optimisation to on the cm2 path. The +coincidence is unlikely. + +Cost to try: 8 sub-blocks times (BM + 2) times a `vec2` = 8320 bytes of workgroup memory +(`dequant_funcs_cm2.glsl:442-444`), which llama.cpp already spends without occupancy trouble on +the same tile. The fetch/store pipelining and the two barriers per 256 k +(`mul_mm_cm2.comp:363-374`, `dequant_funcs_cm2.glsl:486, 522`) come with it. + +**Measurement that proves or kills it.** `harness/vk_gemm_probe.das` already has the instrument: +the `k6x` arm varies one decode callback against spellings of itself with the tile held fixed, +and its `flat` variant is "current compose w/o scale reads" +(`D:\Work\daScript-iquant\modules\dasLLAMA\harness\vk_gemm_probe.das:466-469`). Run `k6x` at the +q6k gate and down shapes it already pins (9728x2560x512 and 2560x9728x512, +`vk_gemm_probe.das:1287-1288`) and read the `nil` to `flat` to shipped ladder. The `flat`-to- +shipped gap is the entire scale-plane cost in TF/s. If it is under 5%, delta 1 is dead. Then add +a k4 twin of that bisect - the file has no k4 scale-plane arm today, only a k4 rate arm +(`vk_gemm_probe.das:357`). +Caveat: arm `DAS_TUNE_MANIFEST` or the checked-in +`performance/last_known_good_sidecar.json` first, or a bare probe run re-execs into a full +retune (`ARCHITECTURE_MEASUREMENT.md:53-56`). + +### Delta 2 - Fold the scale plane back into the quant block, so the decode touches one buffer. Estimated payoff: 5-15%, uniform across formats. + +llama.cpp's callback gets a `buffer_reference` the driver already resolved and reads d, dmin and +all 12 scale bytes as one 16-byte aligned `uvec4` (`dequant_funcs_cm2.glsl:429-431, 461`). Ours +carries two independent address streams and a 20-byte scale stride that is not 16-byte aligned +(`dasllama_vulkan_classes.das:4804-4805, 5034`). Two streams double the distinct addresses in +flight per tile and halve the chance the block-load path coalesces them. + +This one is uniform across formats, which is what would explain the part of the deficit that +IQ4_XS still shows at its otherwise-matched decode-load count. + +**Measurement.** The probe's `cm2x` arm already carries a `push` variant that moves the scale +base off the workgroup read onto a push constant +(`vk_gemm_probe.das:215-216`), and a `flat` variant with no scale read at all +(`:72-74`). Those bracket the address-arithmetic half. The buffer-count half needs a new arm: a +k4 fixture whose `ws` contents are interleaved into the `wq` plane and read through the same +block pointer. Compare TF/s on one shape (d 2048, K 2048, cnt 512, large tile, split off via +`DASLLAMA_CM2_SPLITK=1`). Kill it if the interleaved arm is within 3%. + +### Delta 3 - Split the end-to-end budget before spending anything on the kernel. Estimated payoff: none directly; it decides whether deltas 1, 2 and 4-8 are worth doing at all. + +Every ratio in the table above is end-to-end pp512 tokens per second, not GEMM time. An +end-to-end 0.67x does not prove the GEMM is 0.67x. The chain around it carries 452 nodes and 367 +barriers per window on the 3B (`followup_vulkan.md:246-248`), plus the f16 conversion dispatches +(`dasllama_vulkan_prefill.das:760-771, 852-863, 883-892, 905-914`) and the hazard fences +(`dasllama_vulkan_common.das:2587-2620`). + +**Measurement.** Two halves, both cheap. +On llama.cpp: `set GGML_VK_PERF_LOGGER=1` and run the same model and prompt through +`llama-bench -p 512 -n 0 -ngl 99`. It prints one line per op keyed by shape - +`MUL_MAT q4_K m=2048 n=512 k=2048: N x T us = T us (G GFLOPS/s)` +(`ggml-vulkan.cpp:2278-2297, 2240-2254`). +On ours: the prefill already timestamps every role (`pfq_ts`, `dasllama_vulkan_prefill.das:391`, +roles reported at `:983-985`). Sum the GEMM roles and compare against the sum of llama.cpp's +`MUL_MAT` rows for the same forward. +If our MUL_MAT total is at parity and the wall clock is not, the deficit is outside the GEMM and +deltas 1-2 are the wrong target. If our MUL_MAT total is 0.67x, the deficit is in the kernel and +delta 1 leads. + +### Delta 4 - Pad the activation plane's N to the tile width and keep the fast path. Estimated payoff: zero at exactly 512 tokens, up to about `1/ceil(N/BN)` of the GEMM at other token counts. + +llama.cpp pads B's N to the tile width in the prealloc buffer specifically so the last token +column still takes the unclamped path (`ggml-vulkan.cpp:9319`, gate at +`mul_mm_cm2.comp:345`). Ours sizes the plane to `cnt` and drops the partial column onto the fully +clamped edge path (`dasllama_vulkan_classes.das:4893, 4901`). At 512 tokens with BN 256 there is +no partial column, so this is invisible in the pp512 board - and visible in every real prompt +whose length is not a multiple of 256. + +**Measurement.** Run pp at 512 and at 500 tokens, ours and llama.cpp, same model. Our ratio +should degrade at 500 and theirs should not. If both degrade equally, the edge path is not the +cause. + +### Delta 5 - Narrow the tail accumulator (`enable_smaller_matrices`). Estimated payoff: same trigger as delta 4, roughly half its size. + +llama.cpp allocates a `BN/2` or `BN/4` accumulator for the tail column +(`mul_mm_cm2.comp:36-38, 367, 410`), so a 64-token remainder does not drag a 256-wide +accumulator through the whole k loop. We have no analogue. Fix delta 4 first; this is the +refinement on top of it. + +**Measurement.** Only meaningful after delta 4 lands. Same 500-token comparison, with and without +a narrowed tail stamp. + +### Delta 6 - Reduce the inner unroll from 8 to 4. Estimated payoff: 0 to 5%, sign unknown. + +Ours unrolls 8 deep, 512 k per outer iteration (`dasllama_vulkan_classes.das:4909-4916`); +llama.cpp unrolls 4 at `QUANT_K = 256` (`mul_mm_cm2.comp:341-342`). With an f16 128x256 +accumulator already costing 64 registers per invocation, an 8-deep window the compiler chooses +to software-pipeline could cost an occupancy tier. The architecture doc records that a related +codegen spelling change was worth 27% of prefill throughput on this exact box +(`ARCHITECTURE_GPU_VULKAN.md:126-130`), so codegen sensitivity here is established, not +speculative. + +**Measurement.** Build the k4 large class at unroll 4 and at 2 alongside the shipped 8 and time +all three on one shape through the probe's shape arms. Read the driver's register/occupancy +report if Nsight is attached; otherwise the timing alone decides. + +### Delta 7 - No four-wide decode callback. Estimated payoff: zero today, large the day glslc is upgraded. + +llama.cpp ships a `_v` twin of every decode function returning `f16vec4` +(`dequant_funcs_cm2.glsl:576-623` for Q4_K, `:778-805` for Q6_K, `:1002-1025` for IQ2_XS, and so +on for every format), wired through `GGML_VULKAN_COOPMAT2_DECODE_VECTOR` +(`mul_mm_cm2.comp:82-86`, `vulkan-shaders-gen.cpp:480-484`). The local glslc cannot build it +(`tbo_build_vk.log`), so llama.cpp runs scalar here - but the moment that toolchain moves, +llama.cpp's callback count drops fourfold and every delta above gets 4x harder to close. Our +`[spirv_decode]` surface has no vec4 form. + +**Measurement.** None needed to establish the risk. To size it, build llama.cpp with a glslc that +supports the extension and re-run the same per-format board; the delta between the two llama.cpp +builds is what we would owe. + +### Delta 8 - The per-workgroup prologue and the computed codebooks. Estimated payoff: under 2% on the dense 1B shape; real on MoE grids. + +Five `sched` loads, a workgroup broadcast and a barrier before the first tile load +(`dasllama_vulkan_classes.das:3972-3976, 4849-4851, 4887`), plus synthesising 256 to 2048 +codebook words with arithmetic rather than copying them (`:4861-4886`). On a 32-workgroup dense +dispatch this amortises away. On a MoE expert grid with many small dispatches it does not, which +is the GEMV twin of the problem `followup_vulkan.md:548-557` (entry 35) already flags for tg. + +**Measurement.** A variant that hardcodes the single-region dense case - region 0, workgroup base +0, no `sched` read - A/B against the shipped class on the same shape. If the dense arm gains +under 2%, leave it alone and route the effort to the MoE grids instead. + +**One correction to file regardless of the outcome.** `followup_vulkan.md:540-546` (entry 34) +is wrong on three counts against the repo's own tables: the band is 0.67-0.90x not +0.67-0.70x; "the gap is the shared batch-GEMM tier, not any one format's decode" is contradicted +by Q8 at 100.6%/103.5% (`:211-213`) and by the 62 versus 36-38 TF/s tile rates (`:176-178`, +`:223-224`); and "tg holds parity or better" is contradicted by IQ2_XS tg at 0.54x +(`HOW_TO_ADD_A_FORMAT.md:607`), which entry 35 itself documents. + +--- + +## 4. What `test-backend-ops perf -o MUL_MAT` gives, and the exact flags + +**The binary is already built:** +`D:\Work\llama.cpp\build-vulkan\bin\Release\test-backend-ops.exe` +(`C:\Users\Boris\.claude\jobs\7f9893d9\tmp\tbo_build_vk.log`, final line). + +**Command:** + +``` +D:\Work\llama.cpp\build-vulkan\bin\Release\test-backend-ops.exe perf -o MUL_MAT -b Vulkan0 +``` + +- `perf` selects performance mode (`tests/test-backend-ops.cpp:11013-11014`). +- `-o MUL_MAT` filters by `ggml_op_desc` (`:11021-11027`, usage at `:10992-10993`). +- `-b Vulkan0` filters by `ggml_backend_dev_name`; the Vulkan device name is + `GGML_VK_NAME + index` = `"Vulkan0"` (`:11098`, `ggml-vulkan.cpp:7001`, + `ggml/include/ggml-vulkan.h:10`). +- `--output csv` or `--output sql` for machine-readable rows (`:11040-11047`). +- `-p ` filters on the case's `vars()` string, which for MUL_MAT is + `type_a,type_b,m,n,k,bs,nr,per,k_v,o,src_overlap` (`:4490-4492`). Example: + `-p "type_a=q4_K,.*n=512,k=14336"`. + +**What each row reports:** average microseconds per op, GFLOPS computed as +`2*m*n*k*bs[0]*nr[0]*bs[1]*nr[1]` over wall time, and the run count +(`:4510-4513`, `:1639-1646`). The harness sizes the run count to about 100 GFLOP of work on a +GPU backend, duplicates the op that many times into one graph, does a warmup compute, then loops +whole-graph submits until at least one second has elapsed (`:1569-1601`, `:1620-1636`). + +**What it gives us as a reference row:** a per-kernel GFLOPS number for llama.cpp's cm2 GEMM at a +fixed shape, isolated from the rest of a forward pass, on the same device and driver - the exact +counterpart to `vk_gemm_probe.das`'s per-shape TF/s rows. Because the graph repeats one node, +the f32-to-f16 activation conversion is converted once and reused across the duplicates +(`ggml-vulkan.cpp:9441-9453`), so the reported GFLOPS is close to pure GEMM. + +**The limitation that matters, and the workaround.** Perf mode does not use the correctness test +set; it uses a fixed list built by `make_test_cases_perf` (`:10151`, dispatched at `:10727-10728`). +The only quantized MUL_MAT rows at 512 tokens in that list are + +``` +for (int bs : {1, 2, 3, 4, 5, 8, 512}) + for (ggml_type type_a : all_types) + test_mul_mat(type_a, GGML_TYPE_F32, 4096, bs, 14336, {1,1}, {1,1}); +``` +(`:10321-10323`) + +so the available shape is **m 4096, n 512, k 14336** - a Llama-8B FFN shape, not a 1B/3B one. +`-p` can only filter what is already in the list; it cannot create a 2048x512x2048 or +3072x512x8192 row. Getting our shape needs either a one-line addition to that loop (a local +patch to a tree this task treats as read-only) or `--test-file` with a graph exported by +`test-export-graph-ops` (`:11058-11061`, usage at `:10997`). + +Two practical notes. First, the 4096x512x14336 row is still a useful reference: it takes the +same aligned large pipeline (`tiles_m = 32*4 = 128 > 36`, so `prefer_large`) and split-k 1 +(128 tiles, well over `36*2/3`), which is the configuration a 1B prefill also lands on. Second, +for a shape-matched comparison the cheaper route is `GGML_VK_PERF_LOGGER=1` on a real +`llama-bench -p 512 -n 0 -ngl 99` run: it emits per-shape `MUL_MAT m=.. n=.. k=..` rows +with GFLOPS from the actual model geometry (`ggml-vulkan.cpp:2278-2297`), which is what delta 3 +needs anyway. + +--- + +## Summary table + +| dimension | llama.cpp cm2 | ours | +|---|---|---| +| tile large / medium / small | 128x256x64 / 128x128x64 / 32x64x128 | 128x256x64 / 128x128x64 / 128x32x64 | +| workgroup | 256 invocations | 256 invocations | +| accumulate | f16 (f32 on PREC_F32) | f16, clamped, widened to f32 at store | +| A operand | quantized, decoded in the driver's block load | same | +| B operand | f16, one cached conversion dispatch | f16, one conversion per sibling group, SwiGLU fused | +| scales | inside the block; Q4_K/Q5_K hoisted to shared memory | separate SSBO, re-extracted per element with variable shifts | +| barriers per k-step | 0 (2 per 256 k on Q4_K/Q5_K only) | 0 | +| loads per k-step | 1 A + 1 B | 1 A + 1 B | +| unroll | 4 x BK | 8 x BK | +| split-k | k>=2048, cores/tiles, cap 8, 256-aligned | identical rule | +| N padding | buffer padded to tile width | none (large/medium); small tile rounds to 32 | +| tail accumulator | BN/2, BN/4 | none | +| grid | 3-D from `gl_WorkGroupID` | 1-D plus an SSBO region record | +| batching | `gl_WorkGroupID.z` plus a separate MUL_MAT_ID shader | region list in the same kernel | +| vec4 decode | present, gated on glslc (absent locally) | absent | diff --git a/plans/kernel_parity_research_zen4.md b/plans/kernel_parity_research_zen4.md new file mode 100644 index 0000000000..1bbd1e07b5 --- /dev/null +++ b/plans/kernel_parity_research_zen4.md @@ -0,0 +1,289 @@ +# zen4 grid decode: why llama.cpp wins there and loses on zen2 + +Read-only study of the five codebook formats (IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S; our +kq ids 25/24/23/34/33) at the n=1 decode shape, on AMD EPYC 9R14 (zen4, AVX-512 + VNNI, +our class `x86-vnni512`) against zen2 (3990X, AVX2, class `x86-avx2`). + +Trees read: `D:\Work\llama.cpp` at 6c84c7d5d (`ggml/src/ggml-cpu/arch/x86/quants.c`, +`arch/x86/repack.cpp`, `repack.cpp`, `ggml-cpu.c`, `ggml-common.h`, `ggml-cpu/CMakeLists.txt`, +and the MSVC object `build-clean-cpu/ggml/src/ggml-cpu.dir/Release/ggml-cpu/arch/x86/quants.c.obj`); +`D:\Work\daScript-iquant` (`dasllama_gemm_gen.das`, `dasllama_repack.das`, `llvm_tune.das`, +`llvm_jit_common.das`). Nothing modified in either tree. `x86:N` = `arch/x86/quants.c` line N. + +--- + +## 0. The answer in four lines + +llama.cpp's five grid kernels have **no AVX-512 branch at all** - a znver4 build compiles the +same `#if defined(__AVX2__)` body it compiles on zen2 (x86:2673, 2790, 3088, 3273, 3397). Their +1.42-1.90x zen2 -> zen4 gain is pure microarchitecture on unchanged code: their kernel is a wall +of independent 64-bit table loads, and zen3/zen4 went from 2 loads/cycle to 3 and grew the OOO +window. Their IPC rises from 2.2-3.9 to 3.8-5.9. Ours gains only 1.14-1.26x because the panel +form is **stall-bound, not instruction-bound** - we already execute fewer instructions per weight +than they do and still take longer. + +--- + +## 1. What a zen4 build actually runs + +- Flags: `GGML_NATIVE=ON` -> `-march=native` on gcc/clang (`ggml-cpu/CMakeLists.txt:305-306`); + the local zen2 reference is MSVC (`build-clean-cpu/CMakeCache.txt`: Visual Studio generator, + `/O2`, `GGML_AVX512:BOOL=OFF`) and goes through the FindSIMD probe at CMakeLists.txt:247-296. +- **All five vec_dots are `#if defined(__AVX2__)` / `#elif defined(__AVX__)` / generic.** There + is no `__AVX512F__` and no `__AVX512VNNI__` arm anywhere in them: iq2_xxs x86:2673, iq2_xs + x86:2790, iq2_s x86:3088, iq3_xxs x86:3273, iq3_s x86:3397. On znver4 `__AVX2__` is defined, + so the AVX2 body wins and every vector is 256 bits. +- The one VNNI-aware helper in the file, `mul_sum_us8_pairs_float` (x86:105-119, `_mm256_dpbusd_epi32` + under `__AVX512VNNI__ && __AVX512VL__`), is **not called by any of the five** - they all call + `_mm256_maddubs_epi16` directly (x86:2699, 2887, 3145, 3303, 3464) because the per-block scale + is folded into the following `_mm256_madd_epi16`, which VNNI cannot do. +- **No repack GEMV for any of the five.** `ggml_repack_get_optimal_repack_type` + (`repack.cpp:4528-4722`) covers Q4_0, Q4_K, Q2_K (AVX-512 only, :4627-4632), Q5_K, Q6_K, + IQ4_NL, MXFP4, Q8_0 and nothing else; `grep IQ2\|IQ3` over `repack.cpp` and + `arch/x86/repack.cpp` is empty. `llamafile_sgemm` covers only IQ4_NL among the iq types + (`sgemm.cpp:4131`). Type traits are `nrows = 1`, `vec_dot_type = Q8_K`, `from_float = NULL` + (`ggml-cpu.c:336-366`), so real serving takes the per-row `vec_dot` above. **The grid rows of + the ladder are the fair ones** - unlike k4/k5/k6/q4_0/q8_0/iq4_nl/q2_K. + +--- + +## 2. Instruction shape per 32 weights of ONE row + +Counted from the MSVC `/arch:AVX2` object's inner loops (the zen2 reference binary; the zen4 +build compiles the same source with gcc, so counts shift but the shape does not). + +| fmt | loop | weights/iter | instr | per 32w | mem ops | vec instr | scalar instr | +|---|---|---|---|---|---|---|---| +| iq2_xxs | 0xb1..0x26f | 64 | 85 | 42.5 | 33 | 34 | 51 | +| iq2_xs | 0x120..0x3cb | 128 | 110 | **27.5** | 34 | 100 | 10 | +| iq2_s | 0x150..0x2f2 | 64 | 90 | 45.0 | 26 | 34 | 56 | +| iq3_xxs | 0xc1..0x2e2 | 64 | 96 | 48.0 | 43 | 42 | 54 | +| iq3_s | 0x110..0x39f | 64 | 119 | **59.5** | 31 | 84 | 35 | + +Per 32 weights, by stage: + +- **IQ2_XXS** (x86:2687-2706): 4 `movzbl` index bytes; 4 x u64 `iq2xxs_grid[]` and 4 x u64 + `keven_signs_q2xs[]` reads, all folded into `vmovq`/`vpinsrq` pairs plus 2 `vinsertf128` + (8 `vmovq` + 8 `vpinsrq` + 4 `vinsertf128` per 64 weights); 4 `shr` + 4 `and` for the 7-bit + sign codes; 1 `vpsignb` on the ACTIVATION; 1 `vpmaddubsw`; scale `2*ls+1` as `vmovd` + + `vpbroadcastw` + 1 `vpmaddwd`; 1 `vpaddd`. 16.5 memory ops, of which 8 are 64-bit table reads. +- **IQ2_XS** (x86:2841-2900) is the outlier and the fastest: the index extraction is vectorized + (`_mm256_and_si256(q2_data, m511)`, x86:2842) and the 8th sign bit is a **4-bit parity by + `vpshufb`** (x86:2844-2849, the `k_bit_helper` table) instead of a `ksigns` read. MSVC pulls the + 16 indices back out with 14 `vpextrw` rather than the source's stack union (x86:2823-2824). + Only 8.5 memory ops per 32 weights, 4 of them u64 grid reads. Sign expansion is + `vpshufb`+`vpand`+`vpcmpeqb`+`vpor`+`vpsignb` (x86:2871-2885), 5 vector ops per 32 bytes. +- **IQ2_S** (x86:3120-3151): index arithmetic in the ADDRESS expression (`qs[3] | ((qh[ib32]<<2) + & 0x300)`, x86:3123-3130) - 10 `movzbl`, 11 `shl`, 10 `or`, 8 `and` per 64 weights; 4 u64 grid + reads per 32 weights; signs from an explicit plane byte via `vpbroadcastd`+`vpshufb`+`vpand`+ + `vpcmpeqb` and the cheap **`sub(xor(mask,q8), mask)`** conditional negate (x86:3136) - 2 ops, + not `or(mask,1)`+`vpsignb`. +- **IQ3_XXS** (x86:3287-3310): the most memory ops of the five (24.5 per 32 weights) because its + grid entry is only 4 weights: **8 dword** `iq3xxs_grid[]` reads per 32 weights (16 `movzbl` + + 12 `vpinsrd` + `vmovd` per 64 weights) plus 4 u64 sign reads. +- **IQ3_S** (x86:3429-3471): the most instructions. The 9-bit index is built in a vector + (`_mm256_sllv_epi32` + `_mm256_or_si256`, x86:3433-3438) and then taken back out - MSVC uses 12 + `vpextrd` where the source writes a `__m256i`/`uint32_t[16]` union (x86:3413-3418). 8 dword grid + reads + 12 `vpinsrd` per 64 weights, then the same `sub(xor)` sign negate. + +Everything is 256-bit. Nothing is 512-bit. Nothing uses `vpdpbusd`. + +--- + +## 3. Why zen4 favors that shape, and where our panel loses + +Machine facts (Zen 2 -> Zen 4): macro-op **dispatch stayed 6/cycle**; **loads went 2/cycle -> +3/cycle** and stores 1 -> 2; the ROB grew 224 -> 320; AVX-512 ops are executed as two 256-bit +passes over the same four FP pipes, so a 512-bit instruction has the same bytes/cycle as two +256-bit ones and only saves front-end slots. + +Derived cycles per 32 weights and IPC (clock assumed 4.2 GHz zen2 / 3.7 GHz zen4 - **estimates**; +m=4096 k=14336 = 1.835e6 groups of 32 weights): + +| fmt | ref zen2 us / c32 / IPC | ref zen4 us / c32 / IPC | ours zen2 c32 | ours zen4 c32 | ratio gain ref / ours | +|---|---|---|---|---|---| +| iq2s | 5022 / 11.5 / 3.9 | 3757 / 7.6 / **5.9** | 11.6 | 9.9 | 1.51 / 1.17 | +| iq2xs | 5354 / 12.3 / 2.2 | 3624 / 7.3 / 3.8 | 10.6 | 9.2 | 1.68 / 1.16 | +| iq2xxs | 5103 / 11.7 / 3.6 | 3922 / 7.9 / 5.4 | 11.9 | 9.8 | 1.48 / 1.21 | +| iq3s | 10242 / 23.4 / 2.5 | 6105 / 12.3 / 4.8 | 17.4 | 15.2 | 1.90 / 1.14 | +| iq3xxs | 6573 / 15.0 / 3.2 | 5243 / 10.6 / 4.5 | 15.2 | 12.1 | 1.42 / 1.26 | + +Readings: + +1. **Their kernel was load-issue and window limited on zen2 and is near the dispatch ceiling on + zen4.** IQ2_XXS issues 33 memory ops per 64 weights; at 2 loads/cycle that is a 16.5-cycle + floor against 85 instructions / 6 = 14.2 cycles of dispatch, and it measured 23.4 - short of + both, i.e. window-limited. On zen4 the load floor drops to 11 cycles and it measures 15.8 + against the same 14.2 dispatch bound: **90% of the machine's issue width.** They cannot get + much more from instruction selection on zen4; only from fewer instructions. +2. **We are not instruction-bound - we are stall-bound.** Our iq2xxs panel emits, per row per + superblock, 32 index byte loads + 32 shl + 32 u64 grid loads + 32 lshr + 64 stores (about 24 + instructions per 32 weights, `emit_kq_gather` gemm_gen.das:1585-1616, `iq2_idx2` :1421-1435, + `grid_pair_half` :1381-1385), plus about 7 for the dot and 2 for the sign column: ~33 per 32 + weights against their 42.5. We run **fewer** instructions in **more** cycles - IPC ~3.4 + against their 5.4 (estimate). Nothing zen4 widened is our constraint. +3. **The panel is the only structure we have that they do not.** At mr=16 it is 4096 bytes; the + gather writes 1024 scattered 4-byte stores per superblock-group, one per row into each of 64 + distinct cache lines, visiting the same 64 lines 16 times, and the dot then reads them back as + 64-byte vectors each spanning 16 of those stores (`panel_group_off`, gemm_gen.das:1577-1580; + the reload in `emit_block_iq4xs`). Store issue alone is 512 of the measured ~1253 cycles per + superblock-group. Zen4 doubled store issue (1 -> 2/cycle) and we still gained only 1.21x, so + the cost is the round trip, not the store throughput - a wide load over many narrow stores is + the one direction x86 does not forward. Their own memory round trips (x86:2823 `aux_gindex`, + x86:3413 `index_t`) are wide-store -> narrow-load and carry indices, never weights. +4. **Why the row form (`grid_rows_path`, gemm_gen.das:1469-1479) wins on some formats only.** It + replaces the panel with one u64 grid load per row per 8 weights and a `vmovq`/`vpinsrq`/ + `vinserti128`/`vinserti64x4` tree (`row_weights8` :1749-1759, `concat_rows` :1713-1719). The + whole 8-row group's vector cannot issue until the slowest row's index chain finishes, so the + form trades the panel's stall tolerance for a shorter but **exposed** dependency chain. It wins + exactly where the per-row index is one byte with no arithmetic: + - iq2xxs (1 index byte, `wg_byte` at column 2*blk): 4866 -> 2737 us = 5.5 c32, **ahead of + their 7.9** - already seated for `x86-vnni512` (gemm_gen.das:1476). + - iq3xxs (1 index byte per 4 weights, column signs): 5973 -> 4234 = 1.41x. Seated on + `x86-amx` only (:1477), **not on vnni512**. + - iq2s / iq3s (qs byte + qh bits: `shl`+`and`+`or` on the chain): 4918 -> 3559-3656 and + 7560 -> 6267 - win alone, lose or go mixed at 16 lanes. + - iq2xs (u16 assembled from two byte loads, `iq2xs_word16` :1411-1417, and the sign code in + the same word): 4558 -> 5381 - loses. +5. **SMT flips the verdict.** Two lanes per core share the four FP pipes and one 32 KB L1D. A + stall-bound kernel gets its bubbles filled by the sibling, an instruction-bound one does not, + so at 16 lanes the panel catches up and the row form's extra instructions start to cost: our + one-thread ratios 0.76-0.88 become 0.84-0.90 at the engine shape for four of five formats, and + iq2s's row form "wins alone, loses at 16 lanes" is exactly this. Corollary for every item + below: **instruction cuts pay less at 16 lanes than at one thread, and plane-size increases + cost more** (iq3s already pulls 49 GB/s of the box's 87 at 16 lanes). +6. **The zen2 -> zen4 comparison of THEIR numbers mixes compilers** (MSVC `/arch:AVX2` vs gcc + `-march=znver4`). Part of their 1.42-1.90x is codegen, not silicon. It does not change our + conclusions - ours is measured on our own emitter on both boxes - but do not quote their gain + as a microarchitecture number without rebuilding one side. + +--- + +## 4. Ranked spellings for the vnni512 lattice + +Every item is class-gated, so zen2 keeps what it has unless stated. + +**1. Seat the row form for iq3xxs on `x86-vnni512`.** One line: extend gemm_gen.das:1476 from +`te.kq == 25` to `te.kq == 25 || te.kq == 34`, matching what x86-amx already does (:1477). +Measured on the box: 5973 -> 4234 us, i.e. 0.88x -> **1.24x** of the reference. zen2 untouched +(the class gate). 16 lanes: x86-amx recorded 1.36x at the engine shape for the same change, so +the SMT risk is low. Do this first. + +**2. `gather="qpanel"` - store the iq2 grid entry as ONE qword.** The three iq2 formats read a +u64 grid entry covering 8 weights and then split it into two dwords (`grid_pair_half` +:1381-1385) written to two panel groups 4*mr bytes apart. Re-lay the panel so a row's 8 +consecutive weights are contiguous, store the u64 unsplit, and let the dot read 8-rows-x-8-weights +vectors - the shape `grid_rows_dot` + `pairwise_add_i32` already consume (:1723-1745, :1801). +Per 8 weights the gather goes 1 idx load + 1 shl + 1 grid load + 1 lshr + 2 stores = 6 down to 4, +and **panel stores halve** (64 -> 32 per row per superblock), which is the resource item 3 named. +Targets iq2s, iq2xs, iq2xxs. Estimate 1.2-1.4x on those three at one thread, less at 16 lanes. +No new ISA, no plane change, helps zen2 identically. iq3s/iq3xxs get nothing (their entry is 4 +weights) and stay on item 1. + +**3. Bake the parity-completed sign byte at repack.** `kq_sign_bytes` (:1514-1548) still computes +`ksigns[i] == i | parity(i)<<7` at runtime with a 3-deep XOR/shift chain per `l` per block: +about 40 vector instructions per (blk, qd), 320 per superblock per row group. For **iq2xxs and +iq3xxs it is free** - the aux dword is 4 x 7 sign bits + a 4-bit scale, and the scale already +moved to the scale row, so four full sign bytes fit the same 4 bytes; for **iq2xs** it costs +1 +bit per 8 weights (9 index + 8 sign > 16), so race it. iq2s/iq3s already carry explicit sign +bytes (:1519-1521). Estimate 5-8% on the panel form and more on the row form, on every class and +both shapes; zen2 gains too. + +**4. Mask-register signs at width 512.** `apply_sign_col` (:1552-1564) and `sign_col_pm1` +(:1455-1463) build a byte mask with shuffle + `and` + `icmp` + `sext`, then `xor` + `sub` - six +vector ops per weight vector, each two uops at 512 bits. AVX-512 wants `vptestmb` into a +k-register plus a merge-masked `vpsubb`: two instructions. Emit the compare so LLVM keeps the +mask in `k` (do not `sext` it to `<64 x i8>`). Estimate 3-4 vector instructions saved per weight +vector, ~25% of the dot side. Width-256 path (zen2) unchanged - it keeps `llvm.x86.avx2.psign.b` +(:1730-1732), which is already one instruction. + +**5. Give the grid gemv a 256-bit seat on vnni512.** Measured in the pass's own races: iq2s at 16 +lanes 2528 (maddubs256) / **2383 (vpdpbusd256)** / 2530 (vpdpbusd512) - the crowned 512 seat +loses 6%. Mechanism, not noise: a 512-bit op on zen4 is two 256-bit passes over the same pipes +(no throughput gain), mr=16 doubles the panel to 4 KB per lane (8 KB per core under SMT, a +quarter of L1D), and width 512 forces the `icmp`/`select` sign path instead of the single +`vpsignb`. Blocked on the already-queued "gemv's own crown" item (the gemv currently inherits the +tile's `tune_companion`). Estimate 3-6%, larger at 16 lanes. + +**6. The VBMI symbol lattice - the big one, zen4/Granite/Ice Lake only.** Verified over +`ggml-common.h`: every grid byte comes from a tiny alphabet - `iq2xxs_grid` (:560), +`iq2xs_grid` (:627) and `iq2s_grid` (:758) use exactly **3** distinct bytes {0x08, 0x19, 0x2b}, +and `iq3xxs_grid` (:1017) and `iq3s_grid` (:1052) exactly **8** ({0x04,0x0c,0x14,0x1c,0x24,0x2c, +0x34,0x3e} and the odd 0x01..0x0f). So a grid entry is **16 bits** (8 weights x 2) for the iq2 +family and **12 bits** (4 weights x 3) for the iq3 family, and the entire decode can run in +vectors: +- The grp repack puts a 4-byte column's mr rows contiguous (`repack_iq2xxs_grp`, + dasllama_repack.das:663-700: `kq[dq + (c*mr + r)*4 + t]`), so at mr=16 **one 64-byte load is 64 + grid index bytes** (16 rows x 4 groups). +- For the **256-entry** grids (iq2xxs, iq3xxs) the compact code is two 256-byte byte-planes, and + a 256-byte byte-table lookup is 2 x `vpermi2b` + one blend on index bit 7: ~3 instructions per + 64 indices per plane, ~6-8 for both. +- Expand the 2-bit / 3-bit fields with `vpmovzxbd` + `vpmultishiftqb` (per-byte bit-offset + extraction) + `vpandq` + `vpshufb` against the 3- or 8-entry alphabet: **4 instructions per + 64-byte weight vector**, i.e. 16 rows x 4 weights. +- `ksigns_iq2xs` is `uint8_t[128]` (ggml-common.h:513) - exactly two zmm, so the sign-byte lookup + is ONE `vpermi2b` if item 3 is not taken. +Budget: ~5-7 vector instructions per 64-weight vector against today's ~24 scalar instructions per +32 weights per row. Estimate 2-3x on the five formats at one thread; at 16 lanes the win is +smaller (item 5 of sec.3) but positive, because it removes stores rather than adding traffic. +Prerequisites: add `avx512vbmi` to `TUNE_KNOWN_FEATURES` (llvm_tune.das:1235) and a gate beside +`g_target_x64_avx512vnni` (llvm_jit_common.das:626-628). zen2 declines the seat. +For the **512/1024-entry** grids (iq2xs, iq2s, iq3s) the compact table does not fit `vpermi2b` +(8-16 zmm of table); either carry the 12/16-bit code in the plane instead of the index (+1.33-1.5x +plane bytes, which at 16 lanes trades compute for bandwidth - measure before committing) or leave +those three on items 1-4. + +**7. Do not seat a hardware gather.** llama.cpp measured `_mm256_i32gather_epi32` losing to +`_mm256_set_epi32` **on a Ryzen 7950X, which is zen4**, and left the code commented out with the +note at x86:3440-3442. Zen4 `vpgatherdd` is still microcoded (order 10+ cycles per instruction - +estimate). The column read plus `vpermi2b` (item 6) is the vector path that works here. + +**8. Do not seat `vpdpbssd` or `bias=128` for this class.** Zen 4 has no AVX-VNNI-INT8; those +seats are Zen 5 / Sierra Forest coverage, not a zen4 fix. + +--- + +## 5. Traps + +1. **There is no AVX-512 reference kernel to copy.** All five are `#if defined(__AVX2__)` + (x86:2673, 2790, 3088, 3273, 3397); a znver4 build runs 256-bit `vpmaddubsw`, never + `vpdpbusd`. Anything you find that looks like a 512-bit grid kernel in ggml is for another type. +2. **`_mm256_set_epi64x(grid[a],grid[b],grid[c],grid[d])` is not a vector gather.** It is four + scalar table loads folded into `vmovq`/`vpinsrq` plus a `vinsertf128` - the disassembly shows + exactly that. Transposing it into our layout buys the store, not the loads; the win only + arrives when scalar table reads per weight go DOWN. +3. **Signs-on-the-activation does not port** (already ruled, research_cpu.md sec.8): our lanes are + different rows with different sign patterns while the activation is splatted. llama.cpp's own + arm64 arms put signs on the weights for the same reason. +4. **Their memory round trips run the safe way.** `aux_gindex` (x86:2823-2824, commented "somewhat + hacky, but gives a significant boost") and `index_t` (x86:3413-3418) are ONE wide store feeding + narrow fully-contained loads, and they carry indices. Our panel is the opposite direction and + carries weights. Do not "unify" them. +5. **The iq2_xs `vpshufb` parity trick (x86:2844-2849) is a workaround, not a win for us.** It + exists because x86 cannot vectorize a 128-entry x 8-byte table read; we can bake the + parity-completed byte at repack (item 3) and skip both the table and the parity. +6. **Their scale fold into `vpmaddwd` (x86:2703, 3307, 3468) does not port.** Their scale is one + scalar broadcast over a whole row; ours differs per lane and already rides the repacked scale + row with the global factor (0.125f/0.25f) pre-baked. Porting it is a regression. +7. **Do not reintroduce a horizontal reduction.** They pay `hsum_float_8` per row (x86:2713, 2907, + 3158, 3317, 3478); our accumulator lanes are the output rows. A decode that gets cheaper at the + price of a per-row reduction is a net loss. +8. **`nrc == 1` is asserted in all five** (e.g. x86:2662). There is no multi-row reference grid + kernel - our tile's 6.5-11.9x is against a kernel nobody wrote for that shape, and it is not + evidence about the decode. +9. **The dword column read is format- and form-specific.** `wg_col_byte` (gemm_gen.das:1364-1371) + deliberately falls back to a plain byte load under `te.gridRows` (:1365, "the row form measured + 1.8x slower on the dword read"), while the same read is a win in the panel form for iq3xxs + (:1404, 7862 -> 6602) and iq2xs (:1410, 6501 -> 4702). Do not unify them; race per (format, form). +10. **Count tables per core, not per lane.** Under SMT two lanes share one 32 KB L1D. The iq2s grid + is 8 KB, iq2xs 4 KB, `keven_signs_q2xs` 1 KB, and two mr=16 panels are another 8 KB. Any new + table (item 6 adds 512 B - 1 KB of compact code plus register-resident `vpermi2b` halves) has + to be budgeted against that, and a plane-size increase is charged at the DRAM ceiling + (87 GB/s on the c7a; iq3s already draws 49). +11. **The reference's zen2 and zen4 numbers come from different compilers** (MSVC `/O2 /arch:AVX2` + in `build-clean-cpu`, gcc `-march=native` on the AWS box). Before attributing any part of their + 1.42-1.90x to zen4, rebuild one side with the other toolchain. +12. **iq2_xs is their best format, not their worst** (27.5 instructions per 32 weights against + 42-60 for the rest) because it is the only one whose index extraction and parity are + vectorized. Expect it to be the hardest of the five to pass, and do not read its 0.80x as the + same problem as iq3s's 0.81x. diff --git a/plans/qwen38_thinking_control.md b/plans/qwen38_thinking_control.md new file mode 100644 index 0000000000..1073fee3df --- /dev/null +++ b/plans/qwen38_thinking_control.md @@ -0,0 +1,597 @@ +# How much Qwen 3.8 thinks, and who controls it + +Research memo. Written 2026-09-01. Read-only research; nothing in any checkout was modified. + +Evidence base: +- Qwen's own files, downloaded fresh from Hugging Face into `/tmp` during this session: + `Qwen/Qwen3.8-27B` `chat_template.jinja`, `README.md`, `tokenizer_config.json`, + `generation_config.json`, and `Qwen/Qwen3.8-Flash-Next` `chat_template.jinja`, `README.md`. +- The local llama.cpp checkout `D:\Work\llama.cpp`, at HEAD `6c84c7d5d` ("model: add + Qwen3.8-Flash-Next (qwen4exp) (#27742)"). All `llama.cpp` line numbers below are that commit. +- The local daslang checkout `D:\Work\daScript` (module `modules/dasLLAMA`, server + `utils/dasllama-server`). +- Public docs and papers, cited inline. + +--- + +## Summary in one paragraph + +Qwen 3.8 has exactly one first-party knob for the *amount* of thinking, and it is a prompt +knob, not a budget: `reasoning_effort`, with three legal values `xhigh` (the default), `medium`, +and `low`. The chat template turns that value into one English sentence which it prepends to +the system turn; the model was trained to respond to that sentence. There is a second knob, +`enable_thinking`, but that is on/off, not an amount. Qwen ships **no** token budget, **no** +`thinking_budget` template variable, **no** budget-forcing text, and **no** `/think` `/no_think` +soft switch in the 3.8 template. Everything that actually *caps* the thinking length is bolted +on by the serving stack: llama.cpp has a full reasoning-budget sampler, vLLM has +`thinking_token_budget`, SGLang has a `thinking_budget` that is reported broken. Both +Qwen3.8-27B and Qwen3.8-Flash-Next (`qwen4exp`) ship a byte-identical chat template, so +everything here applies to both without change. + +--- + +## (a) The mechanisms that actually exist + +### A1. `enable_thinking` - a trained-in on/off switch, not an amount + +Template spelling (`Qwen/Qwen3.8-27B/chat_template.jinja`, lines 163-170): + +```jinja +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} +``` + +Two things to notice. + +1. With thinking **on** (the default), the template itself writes the opening `\n` into + the generation prompt. The model never emits ``; it starts already inside the thought + block and only has to emit `` to leave it. This matters for any budget mechanism - + see A6 and B3. +2. With thinking **off**, the template writes an already-closed empty block + `\n\n\n\n`. This is the same "prefill an empty closed block" trick Qwen3 used. + +API spelling, per the model card (`README.md` lines 296, 459): + +```python +extra_body={"chat_template_kwargs": {"enable_thinking": False}} +``` + +The card adds (line 466): on Qwen Cloud the field is top-level `"enable_thinking": False`, not +wrapped in `chat_template_kwargs`. + +Source: and the raw template at +. + +### A2. `reasoning_effort` - the amount knob. It is a system-prompt sentence. + +This is the only first-party control over *how much* the model thinks. Template lines 45-56, +verbatim: + +```jinja +{%- set reasoning_instructions = '' %} +{%- if enable_thinking is undefined or enable_thinking is true %} + {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %} + {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %} + {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }} + {%- endif %} + {%- if resolved_reasoning_effort == 'xhigh' %} + {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} + {%- elif resolved_reasoning_effort == 'low' %} + {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} + {%- endif %} +{%- endif %} +``` + +Five facts follow directly from that code, and they are the most useful things in this memo: + +1. **The whole mechanism is one sentence of English prepended to the system turn.** There is no + special token, no logits change, no counter. The `reasoning_instructions` string is spliced + in ahead of the user's system content (template lines 58-86) - and if the request carries no + system message at all, the template *creates* a system turn holding only that sentence + (lines 81-85). +2. **`medium` injects nothing.** Only `xhigh` and `low` set the string; `medium` leaves it + empty. So a request with `reasoning_effort="medium"` renders to a prompt that is + byte-identical to a request with thinking on and no effort field. `medium` is the neutral + baseline; `xhigh` and `low` are the two deviations from it. +3. **The default is the expensive end.** `reasoning_effort|default('xhigh')` - omitting the + field gets you the "think carefully, validate assumptions, consider alternatives" sentence. +4. **Only three values are legal.** `low`, `medium`, `xhigh`. Anything else - including + OpenAI's own standard `"high"` - makes the template raise, and the request fails. This is a + sharp trap for an OpenAI-compatible front end, because `high` is the value most clients send. +5. Because it is only a prompt sentence, it is **advisory**. The model was post-trained to obey + it, but nothing enforces it, and there is no upper bound on the trace length at any level. + +Model card wording (`README.md` lines 258-262): + +> Qwen3.8 comes with official support for `reasoning_effort`, which can be used to adjust +> reasoning depth and control cost: +> - `xhigh` (default): for complex tasks demanding thorough analysis +> - `medium`: balancing accuracy and speed +> - `low`: efficient reasoning optimizing for speed and cost + +The card also carries an explicit warning against assuming lower effort is always faster +end-to-end (`README.md` line 267): + +> In multi-turn agentic tasks, lower reasoning effort does not always reduce overall task +> completion time. Although it may produce faster per-turn responses, it can also lead to +> insufficient analysis, more failures, and repeated retries, which may increase total latency +> and token consumption. + +On the wire it is a **top-level OpenAI field**, not a template kwarg, in Qwen's own example +(`README.md` line 300): + +```python +completion = client.chat.completions.create( + model="Qwen/Qwen3.8-27B", + messages=messages, + extra_body={"chat_template_kwargs": {"enable_thinking": True, "preserve_thinking": True}}, + reasoning_effort="xhigh", # xhigh by default; supported levels are xhigh, medium, and low + ... +) +``` + +vLLM's own recipe page for this model agrees on all of the above: +. + +### A3. `preserve_thinking` - history retention, which is an amount knob at second hand + +Template lines 111-119: + +```jinja +{%- set reasoning_content = '' %} +{%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} +{%- endif %} +{%- set reasoning_content = reasoning_content|trim %} +{%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} +{%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} +{%- endif %} +``` + +Defaults to true. Note the retention is driven by a `reasoning_content` field on each historical +assistant message - so a client that strips reasoning before storing history silently gets +`preserve_thinking` behaviour with empty thoughts (an empty `\n\n` per turn), +which is neither of the two documented modes. The card claims retention improves KV-cache reuse +and agent consistency (`README.md` line 472). Turning it off shortens the prompt but the card +warns it costs decision consistency in agent loops. + +Note the `or loop.index0 > ns.last_query_index` clause: thinking is *always* kept for assistant +messages after the last real user query, i.e. within the current tool-call chain, regardless of +the flag. + +### A4. What Qwen does *not* ship + +Checked directly against the downloaded files, all negative: + +- **No `thinking_budget` / `reasoning_budget` template variable.** `grep -n "budget" chat_template.jinja` + returns nothing. +- **No budget-forcing text.** The "Considering the limited time by the user, I have to give the + solution based on the thinking directly now." style closer is **not** Qwen's; it is a serving + convention (see A6). +- **No `/think` or `/no_think` soft switch.** Those were Qwen3-generation features documented on + the Qwen3 model cards (). `grep -n "no_think\|/think"` + finds no handling in the 3.8 template and no mention in either 3.8 model card. Whether the 3.8 + weights still react to those strings as plain prompt text is untested - see (c). +- **No special reasoning-length token.** Nothing in the vocab named for effort or budget. + +### A5. Token-level facts you need for any sampler-side work + +From `Qwen/Qwen3.8-27B/tokenizer_config.json`, `added_tokens_decoder`: + +| token | id | `special` flag | +|---|---|---| +| `<|im_start|>` | 248045 | true | +| `<|im_end|>` | 248046 | true | +| `` | 248058 | false | +| `` | 248059 | false | +| `` | **248068** | false | +| `` | **248069** | false | + +`` and `` are **single vocabulary entries**. Forcing an early end of thinking is +therefore a one-token force, not a multi-token string force. (`special: false` here only means +they are not stripped by `skip_special_tokens`; they are still atomic added tokens.) + +`generation_config.json`: `temperature 1.0`, `top_k 20`, `top_p 0.95`, `eos_token_id [248046, 248044]`. + +Recommended sampling per the card (`README.md` lines 252-253) - worth quoting because it differs +between modes: +- Thinking: `temperature=1.0, top_p=0.95, top_k=20, min_p=0.0, presence_penalty=0.0, repetition_penalty=1.0` +- Non-thinking: `temperature=0.7, top_p=0.80, top_k=20, min_p=0.0, presence_penalty=1.5, repetition_penalty=1.0` + +And the agentic sizing guidance (`README.md` lines 507-510), which is the closest Qwen comes to a +budget: *"Reasoning Content: Set the maximum output length to 262,144 tokens. Final Response: Set +the maximum output length to 131,072 tokens."* That is a recommendation to size two separate +caps, not a mechanism. + +### A5b. Qwen3.8-Flash-Next / `qwen4exp` is the same story + +`diff` of the two downloaded templates after trailing-whitespace normalisation: **identical**. +The Flash-Next card says the same thing in prose (`README.md` line 350): *"Qwen3.8-Flash-Next +supports controlling thinking behavior via `enable_thinking`, `preserve_thinking`, and +`reasoning_effort`."* Same three effort levels, same `xhigh` default (line 384). + +The llama.cpp commit `6c84c7d5d` that added `qwen4exp` is purely architectural - hyper-connections, +gated delta net, MoE, PLE n-gram embeddings, UINT64 GGUF arrays. It touches no chat, template, or +reasoning code. `grep -rn "xhigh"` across the whole llama.cpp tree hits exactly one line, the help +text of `--reasoning-effort` (`common/arg.cpp:3711`). There is no Qwen3.8 template fixture under +`models/templates/` (newest Qwen there is `Qwen3.5-4B.jinja`) and no Qwen3.8 test in +`tests/test-chat.cpp`. + +### A6. What serving stacks bolt on + +#### llama.cpp - a real reasoning-budget sampler (the most complete implementation seen) + +`D:\Work\llama.cpp`, HEAD `6c84c7d5d`. + +**The sampler.** `common/reasoning-budget.h:10-16` declares a five-state machine: + +``` +IDLE -> COUNTING -> WAITING_UTF8 -> FORCING -> DONE +``` + +- `IDLE`: passthrough, watching for the start sequence. +- `COUNTING`: decrement per token, watching for a natural end sequence. +- `WAITING_UTF8`: budget spent, but let the current multi-byte character finish. +- `FORCING`: emit the forced sequence token by token. `common/reasoning-budget.cpp:166-186` is + the whole enforcement - every logit except the one forced token is set to `-INFINITY`. +- `DONE`: passthrough. It **re-arms** on a new start tag (`reasoning-budget.cpp:147-161`), + because some models open several `` blocks in one reply. + +**The forced sequence** is `reasoning_budget_message` tokens followed by the first end tag +(`tools/server/server-schema.cpp:415-427`). That is exactly the "Considering the limited time..." +pattern: you supply the sentence, the server prepends it to `` and forces the lot. + +**CLI flags** (`common/arg.cpp`): + +| line | flag | meaning | +|---|---|---| +| 3680 | `--reasoning-format none\|deepseek\|deepseek-legacy` | where thoughts land in the response | +| 3691 | `-rea, --reasoning on\|off\|auto` | sets `enable_thinking` template kwarg | +| 3709 | `--reasoning-effort LEVEL` | passes the string straight into the jinja context | +| 3721 | `--reasoning-budget N` | `-1` unlimited, `0` immediate end, `N` token budget | +| 3729 | `--reasoning-budget-message MESSAGE` | text injected before the end tag on exhaustion | +| 3736 | `--reasoning-preserve` / `--no-reasoning-preserve` | sets the `preserve_reasoning` kwarg | + +Environment aliases exist for each (`LLAMA_ARG_REASONING`, `LLAMA_ARG_REASONING_EFFORT`, +`LLAMA_ARG_THINK_BUDGET`, `LLAMA_ARG_THINK_BUDGET_MESSAGE`). + +**Per-request HTTP fields** (`tools/server/server-common.cpp:1313-1378`): + +- `chat_template_kwargs: {"enable_thinking": bool, ...}` - merged over the server defaults. +- `reasoning_effort: ""` - the OpenAI field. `"none"` is special-cased to + `enable_thinking = false` (line 1326-1328); any other non-empty string is forwarded as the + template kwarg (line 1330). +- `reasoning_budget_tokens`, alias `thinking_budget_tokens` (line 1365-1366), falling back to the + server default. +- `reasoning_budget_message` (line 1375). +- `reasoning_control: bool` (line 1376) - arms the sampler so it can be forced later at runtime. + +**Runtime early-stop.** `POST /v1/chat/completions/control` with +`{"id": , "action": "reasoning_end"}` forces the in-flight completion out of its +thought block mid-stream (`tools/server/server-context.cpp:2431-2447`, documented at +`tools/server/README.md:1454`). It requires `reasoning_control: true` on the original request. +This is a "stop thinking, answer now" button for a UI. + +**Two llama.cpp findings that bear on Qwen 3.8 specifically:** + +1. **The budget arms correctly despite the template pre-filling ``.** Recall from A1 that + the model never emits `` - the template does. llama.cpp handles this: the generation + prompt is re-tokenized into `prefill_tokens` (`common/sampling.cpp:279-292`) and those tokens + are fed through `llama_sampler_accept` on the budget sampler at construction + (`common/sampling.cpp:318-322`), so the `` in the prompt moves the state machine + `IDLE -> COUNTING`. Any stack that rolls its own budget must do the same or the counter never + starts. +2. **`--reasoning-preserve` does not work on Qwen 3.8.** It writes the template kwarg + `preserve_reasoning` (`common/arg.cpp:3742,3745`); the Qwen 3.8 template reads + `preserve_thinking` (template line 116). The names differ, so the flag is inert here. The + working spelling is `chat_template_kwargs: {"preserve_thinking": false}`. + +**How llama.cpp routes the Qwen 3.8 template.** There is no Qwen3.8 branch. The template +contains ``, `` in the +template source (`common/chat.cpp:1172`), which succeeds, so +(`common/chat.cpp:1183-1186`): + +```cpp +data.thinking_start_tag = ""; +data.thinking_end_tags = { "", "" }; +``` + +Both of those flow into the budget sampler. So `--reasoning-budget` works on Qwen 3.8 out of the +box, provided `--jinja` is on. Whether `reasoning_effort` is offered to the client is decided by +a capability probe that renders the template with `reasoning_effort = "low"` and checks whether +the variable was read (`common/jinja/caps.cpp:526-532`); Qwen 3.8 reads it, so it probes as +supported. + +#### vLLM + +- Sampling parameter `thinking_token_budget` - per-request reasoning token limit. +- `--reasoning-parser qwen3` is required for Qwen 3.8, because the template opens every assistant + turn with `` and without the parser the whole thought block lands in `content` + (). +- `--reasoning-config '{"reasoning_start_str": "", "reasoning_end_str": "I have to give the + solution based on the reasoning directly now."}'` - this is where the "limited time" + style closer comes from. The docs say putting transitional language in `reasoning_end_str` + makes the termination "more natural". +- `--default-chat-template-kwargs '{"enable_thinking": false}'` for a server-wide default; + per-request `chat_template_kwargs` always wins. +- The docs list Qwen3, DeepSeek and Nemotron3 as the families supporting a thinking budget. +- Source: . + +#### SGLang + +Exposes `chat_template_kwargs` and `separate_reasoning: true` (which populates +`reasoning_content`), plus a `thinking_budget`. The budget is reported **not enforced** on this +model generation: sgl-project/sglang issue #25536, "thinking_budget not enforced for Qwen3.6 - +reasoning consumes all max_tokens", where `thinking_budget: 200` still produced roughly 1400 +reasoning tokens and left nothing for the answer +(). Treat SGLang's budget as unreliable. + +#### Ollama + +Reported in the model discussions to replace the template with a generic one, which silently +disables `reasoning_effort` entirely +(). + +### A7. Independent write-ups on quantization and on banning reconsideration phrases + +This is directly relevant to us because we run this model quantized. + +**Quantization makes reasoning traces longer.** Two 2026 papers: + +- *"Quantized Reasoning Models Think They Need to Think Longer, but They Do Not"* + (). Abstract, verbatim in the load-bearing part: *"aggressive + PTQ reduces accuracy while increasing chain-of-thought (CoT) length... in up to 52% of the + quantized models' failures, models reach the right answer in intermediate reasoning steps but do + not output it as a final answer... Positions with high KL divergence correlate strongly with high + next-token entropy, and at these positions quantized models disproportionately sample + overthinking markers such as "wait", "but", and "alternatively". We show that simply introducing + a training-free logit penalty on a curated set of overthinking markers can reduce CoT length by + 12--23% while preserving or improving accuracy across 5 models (1.5B-32B parameters), 3 + quantization methods, and 5 benchmarks... Overthinking errors produced by quantized models are + particularly reduced by up to 58%."* +- *"Quantization Inflates Reasoning: Token Inflation as a Hidden Cost of Low-Bit Reasoning Models"* + (). INT4/INT3 can preserve accuracy while inflating reasoning + token count enough to cancel the per-token speedup, with more intermediate steps and more + semantic repetition in the trace. + +The practical reading: a Q4_K_M Qwen 3.8 is expected to think *longer* than the bf16 model at the +same `reasoning_effort`, and the extra length is disproportionately made of "wait" / "but" / +"alternatively" continuations sampled at high-entropy positions. A logit penalty on those markers +is the paper's own remedy and it is cheap. + +**Banning reconsideration phrases outright.** *"Wait, We Don't Need to 'Wait'! Removing Thinking +Tokens Improves Reasoning Efficiency"* (NoWait), - suppresses +explicit self-reflection tokens (`Wait`, `Hmm`) at decode time and reports a 27%-51% shorter +chain of thought across five R1-style model series and ten benchmarks with utility preserved. + +**Rollback and resampling** is the Antislop sampler +(, summary at +). Unlike a logit ban it works on +*multi-token phrases*: it keeps the inference trace, and when a banned pattern completes it +backtracks to the token where the pattern began, downweights the offending continuation, and +resamples. The suppression is soft - `p_new = p_old * 10^(-10s)` with ban strength `s` in `[0,1]`, +`s=1` being a hard ban. It scales to 8000+ patterns (direct token banning caps out near 2000) and +accepts regexes. The cost is real: 69%-96% output slowdown in the bad cases. This is the only +mechanism in the set that can ban a phrase like `", but wait"` precisely - a plain logit ban on +the `wait` token cannot distinguish `", but wait"` from a legitimate `wait`. + +For comparison, llama.cpp itself has **no** backtracking sampler (`grep -rni "antislop\|backtrack"` +over `src/ common/ tools/` finds only unrelated tokenizer and unicode code). It has DRY +(`common/sampling.cpp:353`) and `logit_bias`, which are enough for the NoWait-style single-token +ban but not for phrase-level rollback. + +--- + +## (b) What we could expose as "quick" / "normal" / "high" + +Our stack, for reference: chat rendering is `modules/dasLLAMA/dasllama/dasllama_chat.das` over +per-family `ChatTemplate` data declared in `dasllama_arch_*.das` / +`dasllama_common.das:1120-1150`; sampling is +`modules/dasLLAMA/dasllama/dasllama_sampling.das:23-32` (`SamplingParams`: temp, top_k, top_p, +min_p, repetition/presence/frequency penalties - no logit bias, no forced tokens, no phrase ban); +the OpenAI server is `utils/dasllama-server/openai_server.das`. The design overview is +`modules/dasLLAMA/THINKING.md`. We already carry a Qwen3.8-27B GGUF spec +(`modules/dasLLAMA/performance/model_specs.das:346-351`, `Qwen3.8-27B-UD-Q4_K_M.gguf`, +`serve_ctx = 262144`) but there is **no** `dasllama_arch_qwen38.das` yet, and `chatml_chat` +(`dasllama_common.das:4639-4653`) does not prefill `` on the assistant open, which the +Qwen 3.8 template does. + +The answer to the question is **yes**, and there are three independent levers. I would build them +in this order. + +### B1. Ship `reasoning_effort` as the primary knob (cheap, first-party, no sampler work) + +Map the three rungs straight onto Qwen's three levels: + +| our rung | `reasoning_effort` | what the model sees | +|---|---|---| +| quick | `low` | the "keep your thinking brief and focused" sentence | +| normal | `medium` | no sentence at all (the neutral prompt) | +| high | `xhigh` | the "think carefully, validate key assumptions" sentence | + +Concrete steps: + +1. **Add an effort field to `ChatTemplate`.** The cleanest shape given our existing data-driven + design is a three-slot string set - e.g. `effort_low : string`, `effort_high : string`, with + empty meaning "inject nothing", which is literally how Qwen encodes `medium`. Render it into + the system turn ahead of the user's system content, matching template lines 58-86, including + the case where there is no user system prompt and we must synthesize a system turn holding only + the sentence. +2. **Add a `reasoning_effort` field to the server.** Add `"reasoning_effort"` to `CHAT_FIELDS` + (`utils/dasllama-server/openai_server.das:1556-1559`) - today it is silently warned about as an + unsupported field. Also accept it inside `chat_template_kwargs` (the loop at line 2521-2527 + currently warns on every key but `enable_thinking`). +3. **Decide the out-of-range policy, and do not copy Qwen's.** Qwen's template hard-fails on + `"high"`, which is the single most common value an OpenAI client sends. We should map + `high` -> `xhigh` and `minimal` -> `low`, accept `none` -> thinking off (llama.cpp's rule, + `server-common.cpp:1326`), and reject the rest with a 400 naming the legal set. This is a + deliberate divergence from llama.cpp, which passes the string through and lets the template + blow up. +4. **Keep it a per-family declaration, not a Qwen special case** - some families have no effort + sentence at all and should render nothing regardless of the request. + +Caveat to state in the UI: this is a *request*, not a cap. It has no worst case. + +### B2. Add a hard token budget in the sampler (the only real cap) + +Because `` and `` are single tokens (A5), our version is much simpler than +llama.cpp's general string-sequence machine. + +Concrete steps: + +1. **Add to `SamplingParams`:** `reason_budget : int64` (`-1` off, `0` = end immediately, + `N` = token cap), plus the forced-token list and the optional closer message. +2. **Count inside the block.** The generation loop knows whether it is inside the thought span - + `dasllama_chat.das` already has the reply-side matcher (`ThinkStream` / + `make_think_stream_`/`think_feed_`, `dasllama_chat.das:1054-1057`) for exactly this. Arm the + counter when the block opens. **Critical**: with the Qwen 3.8 shape the block is opened by the + *prompt*, not by a sampled token, so the counter has to be armed at prompt-render time when + `think_open` was prefilled - this is the same bug llama.cpp avoids by feeding the prefill + tokens through the sampler (`common/sampling.cpp:318-322`). +3. **Force on exhaustion.** Set every logit but the forced token to `-inf` for as many steps as + the forced sequence is long. If a closer message is configured, force its tokens first and + `` (248069) last - this is what makes the transition read naturally rather than as an + abrupt cut. Mirror llama.cpp's UTF-8 grace state (`reasoning-budget.h:14`) so we do not sever + a multi-byte character. +4. **Re-arm on a second ``** (`reasoning-budget.cpp:147-161`) - the model can open more + than one block per reply. +5. **Server field:** accept `reasoning_budget_tokens` with alias `thinking_budget_tokens` + (llama.cpp's spelling, `server-common.cpp:1365-1366`) plus `reasoning_budget_message`. + +Suggested rung defaults, to be measured, not assumed: quick 512-1024, normal 4096-8192, high +unlimited. The community reports 22k reasoning tokens on a single SVG prompt at `xhigh` and +~60k per turn in agent use, so "unlimited" for the high rung needs a ceiling from `max_tokens` +regardless (). + +### B3. A "stop thinking now" button (small, high perceived value) + +llama.cpp's `POST /v1/chat/completions/control` with `action: "reasoning_end"` +(`server-context.cpp:2431-2447`) is about thirty lines of work on top of B2: arm the budget +machinery for the request, then let an out-of-band request drive the state machine straight to +FORCING. For an interactive UI watching a model think for four minutes this is worth more than +either of the above. + +### B4. Optional, quantization-motivated: an overthinking-marker penalty + +We serve Q4_K_M, and A7 says that specifically inflates traces with `wait` / `but` / +`alternatively`. A single-token logit penalty is a small addition to `SamplingParams` (we have +no `logit_bias` at all today) and the paper reports 12-23% shorter traces at equal or better +accuracy. Reasonable shape: a per-rung penalty, off at `high`, mild at `normal`, strong at +`quick`. + +Phrase-level bans (`", but wait"`) need rollback and resampling, which we do not have and which +costs 69-96% throughput in the reference implementation. **Do not build that.** The single-token +penalty gets most of the benefit at no throughput cost. + +### B5. Prerequisite work regardless of which rungs we ship + +- **`dasllama_arch_qwen38.das` does not exist.** Qwen 3.8 is ChatML-shaped so `chatml_chat` is + the right base, but three things differ from our Qwen3/3.5 wiring: (i) the generation prompt + must prefill `\n` when thinking is on - our `chatml_chat` does not + (`dasllama_common.das:4642-4643`), and `ChatTemplate` already has the field for it, + `assistant_open_think`, added for gemma-4 (`dasllama_common.das:1135`); (ii) the tool block is + Qwen3-Coder XML (``), **not** the Hermes JSON our + `dasllama_arch_qwen35.das:44` installs; (iii) the effort sentence from B1. +- **History retention conflicts with Qwen's default.** `THINKING.md` records that our `respond_` + stores history **reasoning-stripped**. Qwen 3.8 defaults to `preserve_thinking = true` and + renders `\n{reasoning_content}\n` for every historical assistant turn. Stripping + leaves us rendering empty thought blocks - neither documented mode. We should either carry + `reasoning_content` in history and honour `preserve_thinking`, or render the non-preserving + branch (template line 119) deliberately. Right now we do neither on purpose. + +--- + +## (c) What is uncertain or contradictory + +1. **Nobody publishes what the effort levels actually cost.** Qwen gives three adjectives. The + only numbers found are community anecdotes and they disagree wildly: one report has `medium` + cutting time by about a third with no measurable quality loss, another has `medium` moving + thinking tokens only from 106 to 91 on a short prompt while still cutting latency by a third, + a third has `xhigh` at 22,276 reasoning tokens and 21 minutes on one prompt, another claims + ~60k tokens per agent turn + (, + ). These are different prompts on + different hardware. **We should measure our own three rungs before publishing them.** + +2. **Qwen contradicts itself on whether lower effort is cheaper.** The card promotes `low` as + "optimizing for speed and cost" and then warns that in multi-turn agent work lower effort can + *increase* total latency and token consumption through retries (`README.md:267`). Both can be + true; it means a per-turn benchmark will overstate the benefit of the quick rung for agents. + +3. **`medium` being prompt-identical to "no instruction" is my reading of the template, not a + documented claim.** It follows directly from lines 45-56 (the string is only set for `xhigh` + and `low`), but Qwen never says it. It implies the model's *trained* default behaviour - what + it does with no effort sentence - is the `medium` behaviour, and that `xhigh` being the + template default means the shipped default is deliberately above the trained baseline. That + inference is untested. + +4. **Whether `/think` and `/no_think` still work is untested.** They are absent from the 3.8 + template and both 3.8 cards. The 3.8 weights may still react to them as prompt text, since + they were trained into earlier Qwen generations, but there is no evidence either way and + nothing in the template gives them meaning. Do not build on them. + +5. **The `thinking_budget` name collides across stacks and does not mean the same thing.** + vLLM calls it `thinking_token_budget`; llama.cpp calls it `reasoning_budget_tokens` with a + `thinking_budget_tokens` alias; SGLang calls it `thinking_budget` and reportedly does not + enforce it. A client that sends "the" thinking budget will silently get nothing on two of the + three. If we implement B2, accept every spelling. + +6. **Qwen's own agentic sizing advice assumes a feature most stacks lack.** "Set reasoning to + 262,144 and the final response to 131,072" presupposes *separate* caps for reasoning and + answer. Neither our server nor llama.cpp's has two caps; there is one `max_tokens`. This is + the failure mode SGLang issue #25536 describes - reasoning eats the whole budget and the + answer gets zero. **A budget mechanism (B2) is what makes a single `max_tokens` safe.** + +7. **The llama.cpp `preserve_reasoning` / `preserve_thinking` name mismatch** (A6) is stated + from reading the code, not from running it. `common/arg.cpp:3742` writes `preserve_reasoning`; + the Qwen 3.8 template reads `preserve_thinking` at line 116. I found no llama.cpp code that + translates between the two. Worth a five-minute empirical check before relying on it, and + worth an upstream issue if confirmed. + +8. **The quantization/trace-length papers are recent and I read abstracts, not full method + sections.** The 12-23% and 27-51% figures are the papers' own headline claims on their own + benchmarks, not measured on Qwen 3.8 or on our quant. The direction of the effect is + well-corroborated by two independent papers; the magnitudes are not ours until we measure. + +9. **Not checked:** whether Qwen 3.8 was RL-trained on the effort sentences specifically (the + card says "official support", which implies it, but no technical report was located), and + whether the `qwen4exp` Flash-Next weights behave identically to the dense 27B at the same + effort level despite the identical template. + +--- + +## Sources + +- - model card +- - the template +- +- +- - Flash-Next / `qwen4exp` model card +- - "This model cannot stop thinking" +- - "A crazy thinking model" +- - vLLM recipe +- - vLLM reasoning outputs +- - SGLang thinking_budget not enforced +- - Qwen3 card, for the `/think` `/no_think` history +- - Quantized Reasoning Models Think They Need to Think Longer +- - Quantization Inflates Reasoning +- - NoWait +- and - Antislop rollback sampler +- `D:\Work\llama.cpp` @ `6c84c7d5d` - `common/arg.cpp`, `common/common.h`, + `common/reasoning-budget.{h,cpp}`, `common/sampling.cpp`, `common/chat.cpp`, + `common/jinja/caps.cpp`, `tools/server/server-common.cpp`, `tools/server/server-schema.cpp`, + `tools/server/server-context.cpp`, `tools/server/README.md` +- `D:\Work\daScript` - `modules/dasLLAMA/THINKING.md`, + `modules/dasLLAMA/dasllama/dasllama_common.das`, `dasllama_chat.das`, + `dasllama_sampling.das`, `dasllama_arch_qwen35.das`, + `modules/dasLLAMA/performance/model_specs.das`, `utils/dasllama-server/openai_server.das` diff --git a/skills/LAWS.md b/skills/LAWS.md index 2d5713a478..4e75e466ff 100644 --- a/skills/LAWS.md +++ b/skills/LAWS.md @@ -23,3 +23,8 @@ then there is a race. if nothing available - no race." Profiles are keyed by CPU it), and separate CPU from GPU: "do u want to separate CPU and GPU tunes?" - resolved as kernels-only profiles, runtime knobs never travel. He will remove the dasllama.io tune service if the new-silicon race drops to a couple of minutes. +## 2026-09-01 - skills/tune.md, the `[tune_companion]` section + +The gemv companion gets its own manifest entry, raced among the tile winner's same-layout rows, top two by tile time +only: "i don't see the point of racing anything but top 2"; "im fine with more mints. performance matters more"; +"we split x64-amx from x86-vnni512 for sure". diff --git a/skills/daslang/references/files-and-paths.md b/skills/daslang/references/files-and-paths.md index 40f66c9983..db2bcea301 100644 --- a/skills/daslang/references/files-and-paths.md +++ b/skills/daslang/references/files-and-paths.md @@ -63,8 +63,8 @@ fopen(path, "rb") $(f) { concept-assert raw POD. The array forms **panic above 2 GiB**; `long_fread` / `long_fwrite` return `int64` and have no cap. - `run_and_capture(args, var output, timeout_sec = 0.0) : int` runs a child with no shell, capturing - merged stdout+stderr. **Windows trap: `args[0]` must use backslashes** - the OS will not resolve a - forward-slash relative path - so `replace(exe, "/", "\\")` first. + merged stdout+stderr. A forward-slash `args[0]` spawns on every host - the spawn hands Windows + the backslash spelling its CreateProcess wants. ## Mutating operations and their three error forms diff --git a/skills/tune.md b/skills/tune.md index cb5a7b0fcf..1ca46eef6a 100644 --- a/skills/tune.md +++ b/skills/tune.md @@ -89,6 +89,14 @@ two-function stamp, so a kernel and (say) its repack-layout query can never desync, JIT-time declines included. The sibling is a plain function declared earlier in the same module, with an explicit return type. +A companion may carry its **own** manifest entry, keyed by its function name: +when that entry names a permutation of the same grid that this box can run, +the companion stamps it instead of the main entry's. The harness writes one +only where it measured a different winner for the sibling - the dasLLAMA tuner +gives the kq gemv its own seat among the rows sharing the tile winner's plane +layout - and never for a companion whose choice must follow the main (the +layout query). No entry = the companion follows, as before. + ## The mode contract The `DAS_TUNE_MODE` environment variable selects the compile-time behavior: @@ -174,8 +182,9 @@ fingerprint (and any `version_of=` pin value). Runtime knobs, race tables and box identity never travel - all three are properties of the box that measured them, not of its class. -The class names follow the features `requires=` can gate: `x86-vnni512` / -`x86-vnni256` / `x86-avx2` / `x86-base`, `arm-i8mm` / `arm-neon`; any other +The class names follow the features `requires=` can gate: `x86-amx` (AMX-INT8 + +AMX-TILE over the 512-bit VNNI gate) / `x86-vnni512` / `x86-vnni256` / `x86-avx2` / +`x86-base`, `arm-i8mm` / `arm-neon`; any other architecture gets `-`, a key no shipped profile matches, so such a box always races (`tune_cpu_class()` computes this box's, `tune_class_chain()` its adoption ladder). An untuned auto/restart start walks the ladder from the box's own diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 15997b8eb1..4b5dc782ce 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -1173,11 +1173,19 @@ namespace das { return out; } + static string winNativeExePath ( const char * exe ) { + string s = exe ? exe : ""; + for ( auto & ch : s ) { + if ( ch == '/' ) ch = '\\'; + } + return s; + } + static string winBuildCommandLine ( char ** argv, uint64_t argc ) { string s; for ( uint64_t i = 0; i < argc; ++i ) { if ( i ) s += ' '; - s += winArgvEscape(argv[i]); + s += i ? winArgvEscape(argv[i]) : winArgvEscape(winNativeExePath(argv[i]).c_str()); } return s; } diff --git a/src/builtin/module_builtin_runtime.cpp b/src/builtin/module_builtin_runtime.cpp index b52a590b8e..c7a5f83cb4 100644 --- a/src/builtin/module_builtin_runtime.cpp +++ b/src/builtin/module_builtin_runtime.cpp @@ -2173,6 +2173,7 @@ namespace das if ( strcmp(feature, "avx512bw")==0 ) return os_zmm && (r7[1] & (1<<30)) != 0; if ( strcmp(feature, "avx512vl")==0 ) return os_zmm && (r7[1] & (1u<<31)) != 0; if ( strcmp(feature, "avx512vnni")==0 ) return os_zmm && (r7[2] & (1<<11)) != 0; + if ( strcmp(feature, "avx512vbmi")==0 ) return os_zmm && (r7[2] & (1<<1)) != 0; // AMX names use the LLVM hyphen spelling so cpuid names == target-feature names // (DAS_JIT_X64_FORCE_FEATURES / llc -mattr pass them through verbatim). XCR0 tile // bits are kernel-boot truth; the per-process XTILEDATA grant (Linux arch_prctl) diff --git a/tests/README.md b/tests/README.md index b794fd6f06..25a1c97599 100644 --- a/tests/README.md +++ b/tests/README.md @@ -384,7 +384,7 @@ Every `.das` file in this directory tree is listed below, grouped by subdirector | fio_file.das | File I/O - fopen, fread, fwrite with fuzzing | | | fio_utils.das | File utilities - fexist, rmdir, rmdir_rec, fread/fwrite by path, get_das_version | | | glob_test.das | Pathname glob - `match_glob` (literal, `*`, `**`, `?`, `[a-z]`, `[!abc]`, edge cases), `glob`, `glob_filtered` walk, `is_glob_pattern` | | -| popen_argv.das | `popen_argv` - basic invocation, non-zero exit on unknown flag, exit code capture | | +| popen_argv.das | `popen_argv` - basic invocation, non-zero exit on unknown flag, exit code capture, a forward-slash relative argv[0] | | ## fixed_array/ diff --git a/tests/dastest/test_dasfmt_exclude_mask.das b/tests/dastest/test_dasfmt_exclude_mask.das index eadda48187..567a6d7ef0 100644 --- a/tests/dastest/test_dasfmt_exclude_mask.das +++ b/tests/dastest/test_dasfmt_exclude_mask.das @@ -7,10 +7,7 @@ require daslib/fio [test] def test_dasfmt_exclude_mask(t : T?) { - var exe = get_command_line_arguments()[0] - if (get_platform_name() == "windows") { - exe = exe |> replace("/", "\\") - } + let exe = get_command_line_arguments()[0] let tool = path_join(get_das_root(), "utils/das-fmt/dasfmt.das") var terr : string let tmp = temp_directory(terr) diff --git a/tests/dastest/test_preflight_config.das b/tests/dastest/test_preflight_config.das index ee64a62ef1..e677bf3133 100644 --- a/tests/dastest/test_preflight_config.das +++ b/tests/dastest/test_preflight_config.das @@ -60,10 +60,7 @@ def test_preflight_config(tt : T?) { } tt |> run("--show-help lists the rail-skip flag the skills hardcode") <| @(t : T?) { - var exe = get_command_line_arguments()[0] - if (get_platform_name() == "windows") { - exe = exe |> replace("/", "\\") - } + let exe = get_command_line_arguments()[0] let tool = path_join(get_das_root(), "utils/internal/preflight/main.das") var out : string let rc = run_and_capture([exe, tool, "--", "--show-help"], out, 120.0) diff --git a/tests/fio/popen_argv.das b/tests/fio/popen_argv.das index ec3c9d0023..97f7b79791 100644 --- a/tests/fio/popen_argv.das +++ b/tests/fio/popen_argv.das @@ -112,6 +112,27 @@ def test_popen_argv_arg_with_embedded_quote(t : T?) { } } +[test] +def test_popen_argv_relative_forward_slash_exe(t : T?) { + t |> run("a relative argv[0] with forward slashes and no leading ./ spawns") @(t : T?) { + let exe = get_full_file_name(das_exe()) + let exe_dir = dir_name(exe) + let rel = "{base_name(exe_dir)}/{base_name(exe)}" + let launch_dir = dir_name(exe_dir) + let saved_cwd = getcwd() + if (!chdir(launch_dir)) { + t |> failure("could not chdir to {launch_dir}") + return + } + var output : string + let rc = run_argv([rel, "--help"], output) + t |> success(chdir(saved_cwd), "could not restore the working directory {saved_cwd}") + t |> equal(rc, 0, "spawning {rel} from {launch_dir} failed") + t |> success(find(output, "daslang version") >= 0, + "expected 'daslang version' in output: {output}") + } +} + [test] def test_popen_argv_child_stdin_isolated(t : T?) { t |> run("child cannot read parent's stdin") @(t : T?) { diff --git a/tests/language/cpu_supports.das b/tests/language/cpu_supports.das index 1f406e783b..6f15ac6388 100644 --- a/tests/language/cpu_supports.das +++ b/tests/language/cpu_supports.das @@ -25,6 +25,9 @@ def test_cpu_supports(t : T?) { if (cpu_supports("avx512vnni")) { t |> success(cpu_supports("avx512f"), "avx512vnni implies avx512f") } + if (cpu_supports("avx512vbmi")) { + t |> success(cpu_supports("avx512f"), "avx512vbmi implies avx512f") + } if (cpu_supports("avx512bw")) { t |> success(cpu_supports("avx512f"), "avx512bw implies avx512f") }