From 9de55963e9fc7c9dbefbf3710d01c7d5c064ffcf Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 00:09:42 -0700 Subject: [PATCH 001/123] kernel parity pass, step 0: the kq kernel microbench, the reference rows, and the two research memos kq_kernel_bench.das times every row of a kq kernel's tile and gemv registries at one thread on synthetic planes - the inner loop of kernel work, seconds per try, no model. The reference row is the reference exe's test-backend-ops perf at the same shape; the fact base it produced (the five grid formats at 48-51 ns per superblock against k4/k2's 8.6-8.8, i.e. 0.43x-0.91x of the reference decode kernel) lands in plans/kernel_parity_pass.md with the procedure and the work queue. Two memos (CPU grid vec_dot, Vulkan cm2 mul_mm) name what llama.cpp does that we do not, cited; followup_general 61 and followup_vulkan 34 are rewritten on the measured facts. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 9 + .../benchmarks/matmul/kq_kernel_bench.das | 326 +++++++++ modules/dasLLAMA/followup_general.md | 24 +- modules/dasLLAMA/followup_vulkan.md | 21 +- plans/kernel_parity_pass.md | 102 +++ plans/kernel_parity_research_cpu.md | 670 ++++++++++++++++++ plans/kernel_parity_research_vk.md | 470 ++++++++++++ 7 files changed, 1605 insertions(+), 17 deletions(-) create mode 100644 modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das create mode 100644 plans/kernel_parity_pass.md create mode 100644 plans/kernel_parity_research_cpu.md create mode 100644 plans/kernel_parity_research_vk.md diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 5e2703b677..e2fd0b5c90 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -260,6 +260,15 @@ loses cross-module inlining); the first run after a cache write pays one cold co deser re-key); QUIRK 21 still applies to emitter edits. Numbers, caveats and the invalidation ledger: `plans/jit_compile_time.md`. +**Kernel-loop invocation (adopted 2026-09-01):** a kernel spelling is raced without a model in +`benchmarks/matmul/kq_kernel_bench.das` - `DAS_TUNE_MODE=tune bin/Release/daslang.exe -jit +modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das -- --fmt --perm ` times every +row of the tile's and gemv's `_variants()` registries at one thread on synthetic planes (seconds +per try), and the reference row at the same shape is the reference exe's `test-backend-ops perf +-o MUL_MAT -p "type_a=,type_b=f32,m=4096,n=1,"` under `GGML_BENCH_THREADS=1`. The app run +comes only after a spelling wins there. Procedure, fact base and work queue: +`plans/kernel_parity_pass.md`. + A real file whose every tensor type is now loadable (the header census script in the session scratchpad, or `harness/gguf_dump.das`), through `examples/dasLLAMA/run.das` against `simple_ids.exe` from the llama.cpp reference build for the same prompt; then `test_model_image` diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das new file mode 100644 index 0000000000..8be819d795 --- /dev/null +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -0,0 +1,326 @@ +options gen2 +options persistent_heap +options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) +options _dasllama_internal = true + +require daslib/clargs +require daslib/enum_trait +require dasllama/dasllama_math_gen +require dasllama/dasllama_math +require dasllama/dasllama_kqformat +require daslib/fio +require math +require strings +require daslib/strings_boost + +//! The kq kernel microbench - the inner loop of kernel work. Raw kernels, one thread, no jobque, +//! no model; synthetic planes, because the timing is data-independent. GEMV is the decode shape +//! (one token), TILE the prefill batch. Every row of a kernel's `_variants()` registry is timed: +//! under DAS_TUNE_MODE=tune that is the whole [tune_perm] grid, under normal mode the stamped +//! winner alone. Timing only - a spelling's correctness is gen_tune_probe's TEST mode. +//! Run: DAS_TUNE_MODE=tune bin/daslang -jit modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das -- --fmt iq3s +//! The reference row at the same shape: GGML_BENCH_THREADS=1 test-backend-ops perf -o MUL_MAT -p "type_a=iq3_s" +//! (its default MUL_MAT perf case is m=4096 rows, k=14336, n in {1, 512} tokens - this bench's defaults). + +[CommandLineArgs] +struct BenchArgs { + @clarg_short = "f" + @clarg_doc = "KqFmt names, comma-separated (k4,k5,k6,q40,iq4xs,k3,iq3s,iq3xxs,iq4nl,k2,iq2s,iq2xs,iq2xxs), or all" + fmt : string = "all" + @clarg_doc = "Reduction length - the weight row length (k in test-backend-ops terms)" + n : int = 14336 + @clarg_doc = "Rows - output features (m in test-backend-ops terms)" + d : int = 4096 + @clarg_doc = "Tokens for the TILE arm (the prefill batch); 0 skips the arm" + ntok : int = 512 + @clarg_doc = "Timed repetitions per row; the best one is reported" + rounds : int = 5 + @clarg_doc = "Skip the GEMV (decode) arm" + no_gemv : bool + @clarg_short = "p" + @clarg_doc = "Registry rows to time - comma-separated substrings of the perm suffix (default: every row, the reference body included)" + perm : string + @clarg_doc = "TSV rows only, no banner" + tsv : bool +} + +typedef GemvFn = function<(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void> +typedef TileFn = function<(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void> +typedef LayoutFn = function<() : int> + +let TILE_TOKENS = 4l // tokens one kq tile call covers + +def fmt_of_name(name : string) : KqFmt { + for (f in type) { + if ("{f}" == name) { + return f + } + } + panic("unknown kq format '{name}'") + return KqFmt.q8 +} + +def gemv_variants(f : KqFmt) : array> { + if (f == KqFmt.k4) return <- k4q8_gemv_gen_variants() + if (f == KqFmt.k5) return <- k5q8_gemv_gen_variants() + if (f == KqFmt.k6) return <- k6q8_gemv_gen_variants() + if (f == KqFmt.q40) return <- q40q8_gemv_gen_variants() + if (f == KqFmt.iq4xs) return <- iq4xsq8_gemv_gen_variants() + if (f == KqFmt.k3) return <- k3q8_gemv_gen_variants() + if (f == KqFmt.iq3s) return <- iq3sq8_gemv_gen_variants() + if (f == KqFmt.iq3xxs) return <- iq3xxsq8_gemv_gen_variants() + if (f == KqFmt.iq4nl) return <- iq4nlq8_gemv_gen_variants() + if (f == KqFmt.k2) return <- k2q8_gemv_gen_variants() + if (f == KqFmt.iq2s) return <- iq2sq8_gemv_gen_variants() + if (f == KqFmt.iq2xs) return <- iq2xsq8_gemv_gen_variants() + if (f == KqFmt.iq2xxs) return <- iq2xxsq8_gemv_gen_variants() + var none : array> + return <- none +} + +def tile_variants(f : KqFmt) : array> { + if (f == KqFmt.k4) return <- k4q8_tile_gen_variants() + if (f == KqFmt.k5) return <- k5q8_tile_gen_variants() + if (f == KqFmt.k6) return <- k6q8_tile_gen_variants() + if (f == KqFmt.q40) return <- q40q8_tile_gen_variants() + if (f == KqFmt.iq4xs) return <- iq4xsq8_tile_gen_variants() + if (f == KqFmt.k3) return <- k3q8_tile_gen_variants() + if (f == KqFmt.iq3s) return <- iq3sq8_tile_gen_variants() + if (f == KqFmt.iq3xxs) return <- iq3xxsq8_tile_gen_variants() + if (f == KqFmt.iq4nl) return <- iq4nlq8_tile_gen_variants() + if (f == KqFmt.k2) return <- k2q8_tile_gen_variants() + if (f == KqFmt.iq2s) return <- iq2sq8_tile_gen_variants() + if (f == KqFmt.iq2xs) return <- iq2xsq8_tile_gen_variants() + if (f == KqFmt.iq2xxs) return <- iq2xxsq8_tile_gen_variants() + var none : array> + return <- none +} + +def layout_variants(f : KqFmt) : array> { + if (f == KqFmt.k4) return <- k4q8_layout_gen_variants() + if (f == KqFmt.k5) return <- k5q8_layout_gen_variants() + if (f == KqFmt.k6) return <- k6q8_layout_gen_variants() + if (f == KqFmt.q40) return <- q40q8_layout_gen_variants() + if (f == KqFmt.iq4xs) return <- iq4xsq8_layout_gen_variants() + if (f == KqFmt.k3) return <- k3q8_layout_gen_variants() + if (f == KqFmt.iq3s) return <- iq3sq8_layout_gen_variants() + if (f == KqFmt.iq3xxs) return <- iq3xxsq8_layout_gen_variants() + if (f == KqFmt.iq4nl) return <- iq4nlq8_layout_gen_variants() + if (f == KqFmt.k2) return <- k2q8_layout_gen_variants() + if (f == KqFmt.iq2s) return <- iq2sq8_layout_gen_variants() + if (f == KqFmt.iq2xs) return <- iq2xsq8_layout_gen_variants() + if (f == KqFmt.iq2xxs) return <- iq2xxsq8_layout_gen_variants() + var none : array> + return <- none +} + +//! mr per registry suffix; the layout companion is stamped in lockstep with the tile, so a tile +//! row and the layout row of the same suffix belong together +def layout_mrs(f : KqFmt) : table { + var mrs : table + var lvs <- layout_variants(f) + for (v in lvs) { + mrs[v._0] = int64(invoke(v._1)) + } + delete lvs + return <- mrs +} + +struct Planes { + kq : array + ks : array + xq : array + xs : array + xbs : array + y : array +} + +def sized(var a : array; n : int64) { + a |> reserve(n) + a |> resize(n) +} + +def fill_planes(var p : Planes; f : KqFmt; n, d, ntok : int64) { + let nsb = n / 256l + p.kq |> sized(d * nsb * kq_qsb(f)) + p.ks |> sized(d * nsb * kq_ssb(f)) + p.xq |> sized(ntok * n) + p.xs |> sized(ntok * n / 32l) + p.xbs |> sized(ntok * n / 16l) + p.y |> sized(ntok * d) + var seed = 0x1234u + for (b in p.kq) { + seed = seed * 1664525u + 1013904223u + b = uint8(seed >> 24u) + } + for (b in p.ks) { + seed = seed * 1664525u + 1013904223u + b = uint8(seed >> 24u) + } + for (q in p.xq) { + seed = seed * 1664525u + 1013904223u + q = int8(seed >> 24u) + } + for (s in p.xs) { + s = 0.01 + } + for (bsum in p.xbs) { + bsum = 128 + } +} + +def emit_row(var rows : int&; f : KqFmt; tier, perm : string; n, d, ntok, best_us : int64; nsb : int64; plane_bytes : int64) { + let ns_per_sb = float(best_us) * 1000.0 / float(d * nsb * ntok) + let gbs = float(plane_bytes) / (float(best_us) * 1000.0) + let gflops = 2.0 * float(n) * float(d) * float(ntok) / (float(best_us) * 1000.0) + print("{f}\t{tier}\t{perm}\t{n}\t{d}\t{ntok}\t{best_us}\t{ns_per_sb}\t{gbs}\t{gflops}\n") + rows++ +} + +def bench_gemv(var rows : int&; cfg : BenchArgs; f : KqFmt; var p : Planes) { + let n = int64(cfg.n) + let d = int64(cfg.d) + let nsb = n / 256l + let plane_bytes = long_length(p.kq) + long_length(p.ks) + var vs <- gemv_variants(f) + unsafe { + let kqp = addr(p.kq[0]) + let ksp = addr(p.ks[0]) + let xqp = addr(p.xq[0]) + let xsp = addr(p.xs[0]) + let xbsp = addr(p.xbs[0]) + var yp = addr(p.y[0]) + for (v in vs) { + if (!perm_wanted(cfg.perm, v._0)) { + continue + } + invoke(v._1, yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) // warmup + codegen + var best_us = 999999999l + for (_r in range(cfg.rounds)) { + let t0 = ref_time_ticks() + invoke(v._1, yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) + best_us = min(best_us, int64(get_time_usec(t0))) + } + emit_row(rows, f, "gemv", v._0, n, d, 1l, best_us, nsb, plane_bytes) + } + } + delete vs +} + +def bench_tile(var rows : int&; cfg : BenchArgs; f : KqFmt; var p : Planes) { + let n = int64(cfg.n) + let d = int64(cfg.d) + let ntok = int64(cfg.ntok) + let nsb = n / 256l + let qsb = kq_qsb(f) + let ssb = kq_ssb(f) + let plane_bytes = long_length(p.kq) + long_length(p.ks) + var mrs <- layout_mrs(f) + var vs <- tile_variants(f) + unsafe { + let kqp = addr(p.kq[0]) + let ksp = addr(p.ks[0]) + let xqp = addr(p.xq[0]) + let xsp = addr(p.xs[0]) + let xbsp = addr(p.xbs[0]) + var yp = addr(p.y[0]) + for (v in vs) { + if (!perm_wanted(cfg.perm, v._0)) { + continue + } + let mr = mrs?[v._0] ?? (mrs?["reference"] ?? 0l) + if (mr <= 0l || d % mr != 0l) { + to_log(LOG_WARNING, "kq_kernel_bench: {f} tile '{v._0}' has no layout row (mr={mr}) - skipped\n") + continue + } + let ngroups = d / mr + var best_us = 999999999l + for (r in range(cfg.rounds + 1)) { // the first pass is warmup + codegen + let t0 = ref_time_ticks() + for (g in range64(ngroups)) { + var t0k = 0l + while (t0k < ntok) { + invoke(v._1, yp, kqp + g * mr * nsb * qsb, ksp + g * mr * nsb * ssb, xqp, xsp, xbsp, n, d, g, t0k) + t0k += TILE_TOKENS + } + } + if (r > 0) { + best_us = min(best_us, int64(get_time_usec(t0))) + } + } + emit_row(rows, f, "tile", v._0, n, d, ntok, best_us, nsb, plane_bytes) + } + } + delete vs + delete mrs +} + +def perm_wanted(spec, suffix : string) : bool { + if (empty(spec)) { + return true + } + for (tok in split(spec, ",")) { + let tk = strip(tok) + if (!empty(tk) && find(suffix, tk) >= 0) { + return true + } + } + return false +} + +def formats_of(spec : string) : array { + var fs : array + if (spec == "all") { + for (f in type) { + if (kq_sb(f)) { + fs |> push(f) + } + } + return <- fs + } + for (tok in split(spec, ",")) { + let name = strip(tok) + if (!empty(name)) { + fs |> push(fmt_of_name(name)) + } + } + return <- fs +} + +[export] +def main : int { + var cfg = BenchArgs() + let rc = parse_args_with_help(cfg, "kq_kernel_bench") + return rc if (rc >= 0) + if (!jit_enabled()) { + to_log(LOG_ERROR, "run with -jit: the kq generators only fire under the JIT\n") + return 2 + } + if (cfg.n % 256 != 0 || cfg.ntok % int(TILE_TOKENS) != 0) { + to_log(LOG_ERROR, "--n must be a multiple of 256 and --ntok of {TILE_TOKENS}\n") + return 2 + } + var fs <- formats_of(cfg.fmt) + if (!cfg.tsv) { + print("kq_kernel_bench: n={cfg.n} d={cfg.d} ntok={cfg.ntok} rounds={cfg.rounds} tune_mode={get_env_variable("DAS_TUNE_MODE")}\n") + print("fmt\ttier\tperm\tn\td\tntok\tbest_us\tns_per_sb\tGB/s\tGFLOP/s\n") + } + var rows = 0 + for (f in fs) { + var p : Planes + fill_planes(p, f, int64(cfg.n), int64(cfg.d), int64(max(cfg.ntok, 1))) + if (!cfg.no_gemv) { + bench_gemv(rows, cfg, f, p) + } + if (cfg.ntok > 0) { + bench_tile(rows, cfg, f, p) + } + delete p + } + delete fs + if (rows == 0) { + to_log(LOG_ERROR, "kq_kernel_bench: no rows - nothing matched --fmt '{cfg.fmt}' or every arm was skipped\n") + return 1 + } + return 0 +} diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 97abd94787..bbe9fb84cd 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -761,16 +761,20 @@ `[tune_perm]` spellings and let the probe judge. Done = a per-format note naming what was raced and what won, beside the existing bench rows. -61. **IQ3_S CPU decode: race a no-panel gemv spelling (the 0.92x tail).** The stamped gemv - gathers each superblock into an alloca panel and then runs the vector dot - a store/load - round trip per superblock that a single token never amortizes; llama.cpp's per-row form - (grid words composed straight into vectors, signs applied to the ACTIVATION via - shuffle+cmpeq/xor-sub, magnitudes kept unsigned for maddubs) edges it 57.0 vs 52.4 tg128 - on the zen2 (pp512 is ours 4.93x - the panel amortizes across the tile). The counter to - race as a [tune_perm]: compose the gathered words directly into the weight vectors - (insertelement per i32 lane, no panel), and/or the signs-on-activation form that drops - the abs+psign pair. Done = a gemv perm that takes tg128 at or past llama.cpp's, crowned - by the probe. +61. **Grid-format CPU decode: the panel round trip, one shared cost (kernel-level 0.43x-0.91x).** + Measured 2026-09-01 at one thread, m=4096 k=14336 (`benchmarks/matmul/kq_kernel_bench.das` vs + the reference exe's `test-backend-ops perf`): iq3s 11406 us vs 10340 (0.91x), iq3xxs 11578 vs + 6590 (0.57x), iq2s 11732 vs 5074 (0.43x), iq2xs 11490 vs 5386 (0.47x), iq2xxs 11061 vs 5124 + (0.46x) - all five at 48-51 ns per superblock against k4/k2's 8.6-8.8, a flat cost independent + of the format. Every one of the five gathers ends each decoded dword with a 4-byte store into the + alloca panel and the dot reloads 32 bytes spanning eight such stores - a wide load over narrow + stores, which x86 does not forward; llama.cpp composes grid words into registers on both arches + (`plans/kernel_parity_research_cpu.md`). Signs-on-activation does NOT port to our 8-rows-per-vector + layout (llama.cpp's own arm64 arms sign the weights for the same reason). The spellings, in + order: `gather="reg"` (group-major compose, insertelement per row, no panel), `psign="mask"` (a + vector sign mask replacing GPR sign math + abs), a `vpdpbssd` seat for AVX-VNNI-INT8 hosts, a + repack-baked parity sign byte. Done = each of the five at or past the reference row at one thread, + crowned by the probe; plan and fact base: `plans/kernel_parity_pass.md`. 62. **IQ3_S Metal decode: the ~140 GB/s compose ceiling (tg 0.95x).** Eight GEMV forms raced at n=2048 d=8192 - gather placement x3, gather deleted, signs deleted, llama.cpp's exact diff --git a/modules/dasLLAMA/followup_vulkan.md b/modules/dasLLAMA/followup_vulkan.md index a185a036fd..53e1d14956 100644 --- a/modules/dasLLAMA/followup_vulkan.md +++ b/modules/dasLLAMA/followup_vulkan.md @@ -537,13 +537,20 @@ module) is independent and can land any time - it is pure structure. it; a cell that embeds through the vulkan override and compares against the CPU pool proves it. -34. **END-OF-ARC: the pp512 tier class (~0.67-0.70x of llama.cpp on 1B shapes).** Every - sb-format cm2 tile lands in the same band (k4 control 0.67x, iq3s 0.70x on the 5060 Ti) - while tg and the CPU tiers hold parity or better - the gap is the shared batch-GEMM - tier, not any one format's decode. Boris 2026-08-30: this one bothers him at 0.7 - - schedule a dedicated pass at the END of the iquant-formats arc (after the last format - lands), not per-format. Start from the followup 29-32 streamed-layer levers and a - kernel-level probe of the cm2 tile vs llama.cpp's mul_mm_cm2 at matched shapes. +34. **The pp512 gap on every sb format (0.67x-0.90x of llama.cpp on 1B shapes) - not one band, and + not the tile.** The board spreads 1.34x (IQ4_XS 0.90, Q2_K 0.87, Q3_K 0.80, IQ4_NL 0.78, IQ2_XS + 0.77, IQ3_S/IQ2_S 0.70, IQ3_XXS 0.69, Q4_K 0.67) while the Q8 rows sit at 1.00x+, so the format + decode is in it. The cm2 tile itself matches llama.cpp's design point for point - geometry, + workgroup, decode-in-load callback, one coopMatMulAdd per k-step, no k-loop barriers, the same + split-k heuristic (`plans/kernel_parity_research_vk.md`). The real differences: llama.cpp hoists + Q4_K/Q5_K scales into shared memory as ready (d,m) pairs where ours re-extracts from a second + SSBO with a variable shift per decoded element, and its scales sit in the quant block where ours + ride a separate plane. Order of work: (1) split the end-to-end budget - `GGML_VK_PERF_LOGGER=1` + per-shape MUL_MAT rows against our `pfq_ts` roles decide whether the GEMM is the 30% at all (the + chain carries 367 barriers per window); (2) the scale hoist; (3) interleave the scale plane into + the quant block; (4) pad N to the tile width. `harness/vk_gemm_probe.das` already carries the + isolation arms (`ref` = llama.cpp's own coopmat2 blob in our harness, `k6x flat` = compose + without scale reads). Boris 2026-08-30: this one bothers him at 0.7. Plan: `plans/kernel_parity_pass.md`. 35. **The grid-format GEMV workgroup re-stage is a fixed per-workgroup cost - amplified on small models.** Every u64-grid gemv (iq2s 8 KB, iq2xs 4 KB) stages the codebook into diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md new file mode 100644 index 0000000000..1a595db3a5 --- /dev/null +++ b/plans/kernel_parity_pass.md @@ -0,0 +1,102 @@ +# Kernel parity pass - CPU and Vulkan kernels to same-speed-or-better + +The arc after PR #3915 (the iquant format ladder). The bar is Boris's: **parity is >= 1.0x of +llama.cpp on the row; 0.95x is a gap.** The method is the compile-profile loop below - never a +model load, a mint, or a 48 s JIT re-emit per try. + +Branch `bbatkin/kernel-parity`, stacked on the PR-1 tip; rebases onto master after that merge. + +## 1. The procedure - three rigs, inner to outer + +1. **Inner loop: the kernel microbench, seconds per try, no model.** + `modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das` - raw kernels, one thread, synthetic + planes, every row of a kernel's `_variants()` registry timed. Under `DAS_TUNE_MODE=tune` that is + the whole `[tune_perm]` grid (gemv and tile, since the gemv is the tile's `tune_companion`); + under normal mode the stamped winner alone. `--perm ` narrows the race (on zen2 the two + VNNI seats decline and run the reference body - visible as reference-speed rows). Timing only: + a spelling's correctness is `DAS_TUNE_MODE=test harness/gen_tune_probe.das`. + ``` + DAS_TUNE_MODE=tune bin/Release/daslang.exe -jit modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das -- --fmt iq3s --perm maddubs --ntok 0 + ``` +2. **The reference row, isolated the same way.** llama.cpp's `test-backend-ops perf -o MUL_MAT` + times one mul_mat per quant type; its default case is m=4096 rows, k=14336, n in + {1,2,3,4,5,8,512} tokens - the microbench's default shape. Thread parity through the + `GGML_BENCH_THREADS` define the harness patch carries (re-applied by hand at the reference tip + 6c84c7d5d; `harness/backend_ops_shapes.patch` no longer applies there): + ``` + GGML_BENCH_THREADS=1 D:/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe perf -b CPU -o MUL_MAT -p "type_a=iq3_s,type_b=f32,m=4096,n=1," + D:/Work/llama.cpp/build-vulkan/bin/Release/test-backend-ops.exe perf -b Vulkan0 -o MUL_MAT -p "type_a=q4_K" + ``` + Vulkan has the same rig on our side in `harness/vk_gemm_probe.das` (`ref` arm = llama.cpp's own + coopmat2 SPV blob inside our harness; `k6x flat` = compose without scale reads). +3. **Outer loop: the app, only when a spelling wins rig 1.** `-jit -module-cache + .jitted_scripts/module_cache/.dascache .das -- --jit-split-modules=-1` (5 s warm, + 65 s after a kernel edit); `--tune-only ` re-crowns one family; `DAS_TUNE_POLICY=reference` + is the A/B baseline. The stock monolith invocation runs once per landed kernel for the table + row. `rm -rf .jitted_scripts` after an emitter edit (`dasllama_gemm_gen.das`) - QUIRK 21. + +Rules that ride along: bench at `DAS_JOBQUE_THREADS=16`, tests at 8; one row at a time; one +proven fact per push; research before any kernel edit (the two memos below). + +## 2. The fact base (zen2 3990X, one thread, m=4096 k=14336, us per run; ours = `dot_maddubs_width256_mr8`) + +| format | ours gemv | llama.cpp n=1 | ours/theirs | ours tile n=512 | llama.cpp n=512 | +|---|---|---|---|---|---| +| k4 | 1969 (8.6 ns/sb) | 2421 | 1.23x | 120.9 GFLOP/s | 71.1 GFLOP/s (1.70x) | +| k2 | 2017 (8.8 ns/sb) | 2132 | 1.06x | - | - | +| iq3s | 11406 (49.7 ns/sb) | 10340 | 0.91x | - | - | +| iq3xxs | 11578 (50.5) | 6590 | 0.57x | - | - | +| iq2s | 11732 (51.1) | 5074 | 0.43x | - | - | +| iq2xs | 11490 (50.1) | 5386 | 0.47x | - | - | +| iq2xxs | 11061 (48.2) | 5124 | 0.46x | - | - | + +Reading: the five grid formats cost 48-51 ns per superblock regardless of what each decodes, +against 8.6-8.8 for k4/k2. A flat cost independent of the format is a shared mechanism, not five +decode problems. The model-level tg rows (0.70-0.92x) are this 2x kernel gap hidden behind memory +bandwidth; on the 1B vehicles it shows, on a 27B it mostly does not - but the kernel is the same. + +## 3. Research memos (read before touching the kernels) + +- `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, + x86 and arm64, against our emitter arms. Headline: every one of our five gathers ends each + decoded dword with a 4-byte store into `te.iq3s_panel` and reloads 32 bytes spanning eight such + stores - a wide load over narrow stores, which x86 does not forward. llama.cpp composes grid + words into registers on both arches. The signs-on-activation half of followup 61 does not port + to our 8-rows-per-vector layout; llama.cpp's arm64 arms put signs on the weights for the same + reason. Ranked spellings: (1) `gather="reg"` - group-major compose, `insertelement` per row, + no panel; (2) `psign="mask"` - a vector sign mask instead of GPR sign math + `llvm.abs`; + (3) a `vpdpbssd` seat (AVX-VNNI-INT8 hosts, no generator change); (4) repack-baked parity sign + byte (free for IQ2_XXS/IQ3_XXS). Incidental waste: IQ2_XXS reads its aux word as four i8 loads + + shifts where IQ3_XXS uses one i32 load; IQ2_XS builds a u16 from two i8 loads. +- `kernel_parity_research_vk.md` - llama.cpp's cm2 `mul_mm` against our cm2 tile. Headline: the + same kernel design (tile geometry, workgroup, decode-in-load callback, one coopMatMulAdd per + k-step, zero barriers in the k loop, the same split-k heuristic) - the gap is not the tile. + Real differences: llama.cpp hoists Q4_K/Q5_K scales into shared memory as ready `(d,m)` pairs + (`shAscales`), ours re-extracts from a second SSBO with a variable shift per decoded element; + its scales sit in the quant block, ours in a separate plane. Ranked: (1) split the end-to-end + pp512 budget first - `GGML_VK_PERF_LOGGER=1` per-shape MUL_MAT rows against our `pfq_ts` roles + decides whether the GEMM is even the gap (367 barriers per window in the chain); (2) the scale + hoist (10-20% on k4/k5); (3) interleave the scale plane into the quant block; (4) pad N to the + tile width; then narrow-tail accumulator, unroll, vec4 decode. Corrections: followup_vulkan 34's + "same band" (it is 0.67-0.90x), "shared tier not decode" (Q8 rows sit at 1.00x+), and "tg holds + parity" (IQ2_XS 0.54x) are stale. glslc here lacks NV_cooperative_matrix_decode_vector, so + llama.cpp runs its scalar decode too - apples to apples today, and its vec4 twin is a widening + waiting to happen. + +## 4. Work queue (one item at a time; a row is done at >= 1.0x on rig 2's shape AND the vehicle) + +CPU decode (gap 2): +1. `gather="reg"` spelling on the shared gather emitter - one change, five formats; race in rig 1. +2. `psign="mask"`; 3. the IQ2_XXS / IQ2_XS load waste; 4. repack-baked parity sign; 5. retro + audit of IQ4_XS/Q3_K per followup 60 (already >= 1.0x; low priority). + +Vulkan pp (gap 1): +1. Budget split (measurement, no kernel edit) - is the GEMM the 30%? 2. `shAscales`-style scale + hoist; 3. scale plane interleave; 4. N padding; 5+. the rest of the memo's list. + +Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. + +## 5. Ledger + +- 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base + above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. diff --git a/plans/kernel_parity_research_cpu.md b/plans/kernel_parity_research_cpu.md new file mode 100644 index 0000000000..dcff040f4d --- /dev/null +++ b/plans/kernel_parity_research_cpu.md @@ -0,0 +1,670 @@ +# CPU grid-quant dot: what llama.cpp does, what we do, and the gap + +Read-only study of five codebook ("grid") formats — IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S — +plus Q2_K as the near-parity control, for the single-token decode shape (GEMV). + +Trees read: + +- `D:\Work\llama.cpp` — `ggml/src/ggml-cpu/arch/x86/quants.c`, `arch/arm/quants.c`, + `quants.c`, `ggml-cpu.c`, `repack.cpp`, `llamafile/sgemm.cpp`, `ggml/src/ggml-common.h` +- `D:\Work\daScript-iquant` — `modules/dasLLAMA/dasllama/dasllama_gemm_gen.das`, + `dasllama_math_gen.das`, `dasllama_repack.das`, `modules/dasLLVM/daslib/llvm_tune.das` + +Nothing was modified in either tree. + +--- + +## 0. The one structural fact that frames everything + +llama.cpp and we solve a *different shape*, and almost every technique difference follows +from that. + +**llama.cpp is one row at a time.** `ggml_vec_dot_iq3_s_q8_K(n, s, bs, vx, bx, vy, by, 1)` +walks one weight row against one activation row. Thirty-two consecutive weights of that one +row form one 256-bit register. Because the activation vector is *the same 32 bytes* the row +is being dotted against, llama.cpp is free to move the per-weight sign onto the activation. +Each row ends with a horizontal float reduction (`hsum_float_8`, x86 quants.c:2713) or, on +ARM, with a cross-lane `vaddvq_s32` per 32-weight sub-block (arm quants.c:3678). + +**We are `mr` rows at a time, activation broadcast.** `setup_tile_emit` +(gemm_gen.das:2123-2141) sets `rv = width/32` and `rq = mr/rv`; one weight vector holds +**4 bytes from each of `rv` different output rows**. `dot_lane` (gemm_gen.das:328-366) splats +one dword-group of the activation across every lane and dots. For the seated x86 perm +(`width = 256, mr = 8`) one `<32 x i8>` weight vector = 8 rows × 4 weights; for AVX-512 +(`width = 512, mr = 16`) it is 16 rows × 4 weights; on NEON (`width = 128, mr = 8, rq = 2`) +it is two vectors of 4 rows × 4 weights. + +Consequences, both directions: + +- **We win the reduction.** Our accumulator lanes *are* the output rows. There is no + horizontal sum anywhere in the kernel — `emit_block_iq4xs` ends with one `sitofp` + one + FMA per superblock per row-lane (gemm_gen.das:1943-1953). llama.cpp's ARM kernels pay a + `vaddvq_s32` per 32-weight block per row (arm:3678, 3845, 3911, 4021), i.e. 8 cross-lane + reductions per superblock per row on a serial dependency chain. This is almost certainly + why our prefill/tile numbers on M1 are 2.1x-6.4x while decode lags. +- **We cannot put signs on the activation.** The 8 lanes of one weight vector belong to 8 + *different* rows with 8 *different* sign patterns, while the activation dword is shared + across all lanes. Signs must go on the weights. This is a hard consequence of the layout, + not a choice, and it is the part of followup entry 61 that needs correcting (see §8). +- **Grid decode cannot stream.** A grid word is 4 or 8 bytes of *one* row. In llama.cpp's + layout, 8 consecutive grid words of one row *are* the next vector, so the composition + `_mm256_set_epi32(grid[i7],...,grid[i0])` writes the register directly. In our layout a + weight vector needs one grid dword from each of 8 different rows — which is *also* exactly + a `set_epi32` of 8 table reads. The composition is available to us; we just do not use it. + Today we round-trip through an `alloca` panel instead. + +--- + +## 1. llama.cpp's mul_mat path selection for these types + +`ggml_compute_forward_mul_mat` (ggml-cpu.c:1254). + +- `src1` (activations) are quantized to the row type's `vec_dot_type` **once**, cooperatively + by all threads, into `params->wdata` (ggml-cpu.c:1321-1355). For all six formats + `vec_dot_type = GGML_TYPE_Q8_K` (ggml-cpu.c:298-366). +- `nrows = 1` for all six (ggml-cpu.c:302, 338, 344, 350, 356, 362), so + `num_rows_per_vec_dot` is 1 (ggml-cpu.c:1435) — no 2-row MMLA fast path, unlike Q4_K/Q6_K + which get `nrows = 2` under `__ARM_FEATURE_MATMUL_INT8` (ggml-cpu.c:313-317, 331-335). +- Work is chunked over rows and dispatched to + `ggml_compute_forward_mul_mat_one_chunk` (ggml-cpu.c:1164), which calls `vec_dot` per row. +- **`from_float` is NULL for all five grid formats** (ggml-cpu.c:337, 343, 348-349, 356, 362) + — these types are never produced on the fly, only read. + +**Repack / "extra buffer type": none of the five grid formats has one.** +`ggml_repack_get_optimal_repack_type` (repack.cpp:4528) covers Q4_0, Q8_0, Q4_K, Q5_K, Q6_K, +MXFP4, IQ4_NL, and **Q2_K** — nothing else. So for IQ2_XXS/IQ2_XS/IQ2_S/IQ3_XXS/IQ3_S, +decode is *always* the per-row `vec_dot` above. + +**Q2_K is the exception and it matters for the control measurement.** repack.cpp:4627-4632 +selects `q2_K_8x8_q8_K` when `ggml_cpu_has_avx512()` and `ne[1] % 8 == 0`; the GEMV entry is +`gemv` (repack.cpp:3976-3984) → `ggml_gemv_q2_K_8x8_q8_K`. +That is an 8-row-interleaved GEMV — *our* shape. There is also a RISC-V `q2_K_16x1_q8_K` +(repack.cpp:4633-4642). So a Q2_K comparison on zen2 (no AVX-512) races +`ggml_vec_dot_q2_K_q8_K`, and the same comparison on zen4 races the repacked 8x8 gemv. Worth +keeping straight when reading the 1.02x/0.95x control numbers. + +`llamafile_sgemm` (ggml-cpu.c:1293-1318, 1364-1385) covers only F32, BF16, F16, Q8_0, Q4_0, +Q5_0, IQ4_NL (sgemm.cpp:3829, 3893, 3957, 4041, 4078, 4115, 4131) — none of the six. + +--- + +## 2. Grid and sign tables (shared by both trees) + +- `iq2xxs_grid` `uint64_t[256]` (ggml-common.h:560), `iq2xs_grid` `uint64_t[512]` (:627), + `iq2s_grid` `uint64_t[1024]` (:758) — 8 weights per entry. +- `iq3xxs_grid` `uint32_t[256]` (:1017), `iq3s_grid` `uint32_t[512]` (:1052) — 4 weights per + entry. +- **Every grid byte is a small non-negative magnitude**: iq2xxs bytes are `{0x08,0x19,0x2b}` + (ggml-common.h:561-566), iq3xxs `0x04..0x3e` (:1018), iq3s `0x01..0x0f` (:1053). Two + things follow: (i) the magnitudes are legal `u8` operands for `VPMADDUBSW`/`VPDPBUSD`, and + the worst-case pair sum `2 × 62 × 127 = 15748` cannot saturate `i16`; (ii) no magnitude is + zero, so per-byte two's-complement negation inside a 32-bit word never carries across a + byte lane — a fact both trees rely on (ours explicitly, gemm_gen.das:1477-1479). +- `ksigns_iq2xs` `uint8_t[128]` (ggml-common.h:513): 7 sign bits → the 8-bit sign byte with + bit 7 set to the parity completion. `kmask_iq2xs[8] = {1,2,4,...,128}` (:509). +- `keven_signs_q2xs` `int8_t[1024]` = 128 × 8 `±1` bytes, defined *per arch file* + (x86:2624, arm:3595, loongarch:1577, powerpc:1388). Read as `const uint64_t * signs64` + (x86:2675, arm:3646) so one 64-bit load gives 8 `±1` bytes. + +--- + +## 3. IQ2_XXS + +Packing: per 32 weights, four 8-bit grid indices + a 32-bit aux word carrying four 7-bit +sign codes and a 4-bit scale in the top nibble. + +### (a) llama.cpp x86 (AVX2), `ggml_vec_dot_iq2_xxs_q8_K`, quants.c:2660-2714 + +1. Staging: `memcpy(aux32, q2, 16)` into a 16-byte scalar array, aliased as `aux8` + (x86:2677-2678, 2690). Scalar staging of *packed bytes only* — never of decoded weights. +2. Grid compose: `_mm256_set_epi64x(iq2xxs_grid[aux8[3]], ..., aux8[0])` (x86:2691-2692) — + four 64-bit table reads written straight into a ymm. No gather instruction, no store. +3. Signs: `signs64[(aux32[1] >> 21) & 127]` etc. (x86:2693-2696) — four 64-bit reads of + `keven_signs_q2xs` composed into a second ymm of `±1` bytes. The parity-completed 8th + sign is *in the table*; no parity arithmetic at runtime. +4. Sign application: `_mm256_sign_epi8(q8_1, s2_1)` (x86:2697-2698) — **applied to the + activation**, one instruction per 32 bytes. Grid magnitudes stay unsigned. +5. Dot: `_mm256_maddubs_epi16(q2_1, q8s_1)` (x86:2699-2700), u8 × s8 → i16 pairs. +6. Scale: `_mm256_madd_epi16(dot, _mm256_set1_epi16(2*ls+1))` (x86:2703-2704) — the per-32 + scale is folded into the *same* `vpmaddwd` that does the i16→i32 pair reduction. One + instruction does scale and widening together. +7. Accumulate: two independent i32 ymm chains `sumi1`/`sumi2` (x86:2705-2706), added once + per superblock, `cvtepi32_ps` + `fmadd` with `d = x.d * y.d` (x86:2709). The `0.125f` + global factor is applied once at the very end (x86:2713). + +No intermediate weight buffer. Only the 16-byte `aux32` staging array touches memory. + +An SSE/AVX-only fallback exists (x86:2715-2770) using `_mm_set_epi64x` pairs. + +### (b) llama.cpp arm64 (NEON+dotprod), quants.c:3631-3689 + +1. Same 16-byte `aux32` staging (arm:3644-3645, 3663). +2. Grid compose: `vcombine_s8(vld1_s8(iq2xxs_grid + aux8[0]), vld1_s8(... + aux8[1]))` + (arm:3664-3667) — two 8-byte table loads combined into a q-register. NEON's equivalent of + `set_epi64x`. +3. Signs: same shape from `signs64` (arm:3668-3671). +4. Sign application: `vmulq_s8(q2u.val[i], q2s.val[i])` (arm:3672-3675) — **applied to the + weights**, because `sdot` is signed×signed and there is no `u8×s8` NEON dot. This is the + arch-forced mirror of the x86 choice. +5. Dot: `ggml_vdotq_s32` chained twice per 32 weights (arm:3676-3677). +6. Scale + accumulate: `sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[1] >> 28))` + (arm:3678-3679) — a **cross-lane reduction and a float multiply-add per 32-weight + block**, 8 per superblock. This is the weakest part of llama.cpp's ARM grid kernels. +7. `*s = 0.25f * sumf` (arm:3684). + +### (c) What ours does today + +Seat: `dasllama_math_gen.das` — the `iq2xxsq8_tile_gen` block, four perms only +(`mr=8` NEON sdot; `maddubs/256/8`; `vpdpbusd/256/8`; `vpdpbusd/512/16`), same list as every +other grid format. GEMV generator: `iq2xxs_gemv_gen` → `kq_gemv_gen_impl(gc, 25)` +(gemm_gen.das:3344). + +- Panel alloca `256 * mr` bytes, `align 32`, one per GEMV call + (gemm_gen.das:3081-3082). +- Per superblock, `emit_iq2xxs_gather` (gemm_gen.das:1749-1806) runs an **`mr`-iteration row + loop**; inside, for each of 8 blocks and 4 sub-lanes: + - the 4-byte aux word is read as **four separate `i8` loads plus three shifts and three + ORs** (gemm_gen.das:1766-1770) — note IQ3_XXS reads the identical field with **one + aligned `i32` load** (gemm_gen.das:1438-1439). The offsets `(2*blk+1)*mr*4 + 0..3` are + contiguous and 4-aligned, so the four-load spelling is pure waste. + - `ksigns` lookup: one `i32` load from the `[128 x i32]` global (gemm_gen.das:1633-1687, + 1775-1776). + - grid: two `i32` loads from the `[1024 x i32]` low/high-word-pair global + (gemm_gen.das:1782-1784). + - sign apply: `w = (gw ^ mw) + (mw & 0x01010101)` where `mw` is an `i32` from a + 16-entry nibble→byte-mask table (gemm_gen.das:1785-1787) — **three GPR ops and two loads + per 4 weights**, i.e. ~40 scalar ops per 32 bytes of weight. + - store: one 4-byte store into the panel (gemm_gen.das:1793-1794). +- The dot then reloads the panel as `<32 x i8>` vectors (gemm_gen.das:1880-1883), takes + `abs_w` to recover the magnitudes it just threw away (gemm_gen.das:1888-1889, + 370-375 — `llvm.abs` → `VPABSB`), signs the *splatted activation* with `VPSIGNB` + (gemm_gen.das:349-357), then `maddubs` + `pmaddwd` (gemm_gen.das:361-365). +- Scale: per-32 `i8` scale row loaded as `<8 x i8>`, `sext` to `<8 x i32>`, then a **`vpmulld` + per block per row-lane** (gemm_gen.das:1922-1924, 1935). Integer accumulation across the + superblock, one `sitofp` + FMA at the end (gemm_gen.das:1943-1953). The `0.25f` global + factor is pre-baked into the repacked scale row, so it costs nothing at runtime. + +**The memory shape is the wrong way round.** The store address is +`panel + r*4 + (blk*4 + g)*mr*4 + half` (gemm_gen.das:1793); consecutive stores for one row +are `mr*4 = 32` bytes apart, and the reload at +`(blk*16 + j*4)*mr` (gemm_gen.das:1875) is a 32-byte load covering **8 distinct 4-byte +stores from 8 different loop iterations**. A wide load spanning several narrow stores is the +one direction x86 store-to-load forwarding cannot take. llama.cpp does round-trip through +memory — but only for *indices*, and only in the safe direction (§6c): one wide store, then +narrow fully-contained loads. + +Rough uop budget per 32-byte weight vector (mr = 8, AVX2): +ours ≈ 8×(2 loads + 3 ALU + 1 store) + 1 reload + `abs` + `psign` ≈ 51; +a register-composed form ≈ 9 (`vpinsrd` with folded grid load) + 9 (mask) + `vpor` + +`vpsignb` ≈ 20. + +### (d) Deltas, ranked + +| # | Delta | Candidate seat | Why / estimated payoff | +|---|---|---|---| +| 1 | **Delete the panel.** Compose the 8 grid dwords of one weight vector directly with `insertelement` (the `set_epi32` shape) instead of storing them and reloading. | `tune_perm(dot="maddubs", width=256, mr=8, gather="reg", requires="avx2")` and the `vpdpbusd` twins | Removes 512 scattered 4-byte stores + 64 wide reloads per superblock per row-group, and the store-forwarding hazard. Biggest single item; ~2.5x fewer decode uops. | +| 2 | **Sign in vectors, not GPRs.** Compose a second vector of the 0x00/0xFF byte masks and apply `vpor(mask,1)` + `vpsignb` to the activation, keeping the raw grid vector as the unsigned `maddubs` operand. Deletes both the 3-GPR-op-per-dword negate *and* `llvm.abs`. | folded into `gather="reg"`, or split as `psign="mask"` | ~24 GPR ops → 2 vector ops per 32 bytes; also removes `VPABSB` from the inner loop. | +| 3 | **One `i32` load for the aux word.** Replace the four `i8` loads + 3 shifts + 3 ORs at gemm_gen.das:1766-1770 with the single aligned `i32` load IQ3_XXS already uses (gemm_gen.das:1438-1439). | no seat needed — a straight bug-shaped fix | 6 uops × 8 blocks × mr rows per superblock, for free. Do this regardless. | +| 4 | **Bake the parity-completed sign byte at repack time.** The aux word is 4×7 sign bits + a 4-bit scale, and the scale already moved to the scale row (gemm_gen.das:1746-1748) — so 4 full sign *bytes* fit in the same 4 bytes. Deletes the `ksigns` load entirely and makes the sign bits plane-resident like IQ3_S's. | repack-side; pairs with delta 2 | Removes one dependent table load per 8 weights. Zero size change for this format. | +| 5 | **Seat `vpdpbssd`.** The panel bytes are already signed and the activation is signed; `dot_lane` takes that path with no `abs` and no `psign` (gemm_gen.das:344-347, 370-372). Currently unseated for every grid format. | `tune_perm(dot="vpdpbssd", width=256, mr=8, requires="avxvnniint8")` | One-line seat, zero generator change. Only helps AVX-VNNI-INT8 hosts (not zen2/zen4). | +| 6 | **Seat `bias=128`.** Under `gather="reg"` the biased byte is free — `((mag^mask) + (mask&0x01010101)) ^ 0x80808080` — and `bias128` drops `abs` and `psign` on VNNI hosts. Blocked today: `setup_tile_emit` forces `te.bias = 0` whenever `te.kq != 0` (gemm_gen.das:2136). | `tune_perm(dot="vpdpbusd", width=256, mr=8, bias=128, gather="reg", requires="avxvnni\|avx512vnni")` | Needs the `te.bias` clamp lifted for panel formats. Not valid under `maddubs` — `(w+128)×x` pair sums saturate `i16`. | +| 7 | **Fold the scale into `pmaddwd`.** llama.cpp gets scale + widening in one `vpmaddwd` (x86:2703). We cannot broadcast a scalar scale (each lane is a different row), but we can keep an `<16 x i16>` per-row scale vector and use `pmaddwd` against the `maddubs` output instead of `vpmulld` on the i32 accumulator. | `scale="madd16"` | Replaces a 10-cycle `vpmulld` per block per row-lane with a 5-cycle `vpmaddwd` that also removes the separate ones-vector `pmaddwd`. Modest but cheap. | + +--- + +## 4. IQ2_XS + +Packing: per 32 weights, four `uint16` words, each = 9-bit grid index + 7 sign bits; the 8th +sign is parity. Per-16 scales in a nibble-packed `scales[QK_K/32]`. + +### (a) llama.cpp x86 (AVX2), quants.c:2778-2900 + +1. **Vectorized index extraction.** One 32-byte load of eight `uint16` words; `aux_gindex = + _mm256_and_si256(q2_data, m511)` (x86:2842). The result is written to a stack union and + read back as `uint16 gindex[16]` (x86:2823-2824, commented *"somewhat hacky, but gives a + significant boost in performance"*). +2. **Parity by `pshufb`.** `partial_sign_bits = q2 >> 9`, `partial_sign_bits_upper = q2 >> 13`, + XOR them, then `_mm256_shuffle_epi8(bit_helper, ...)` with a 16-entry nibble table that + returns 0x00/0x80 (x86:2844-2849). That is a 4-bit population-parity computed as a + single `vpshufb`, giving the 8th sign bit in bit 7. No `ksigns` table read at all on this + arch. +3. Grid compose: four `_mm256_set_epi64x` of four `iq2xs_grid[...]` reads each + (x86:2856-2863). +4. **Sign expansion by shuffle + cmpeq.** For each 32-byte group: `vpshufb` broadcasts the + group's sign byte across 8 lanes (`block_sign_shuffle_1/2`, x86:2796-2803), `AND` with + `bit_selector_mask` = `{1,2,4,...,128}` repeated (x86:2804-2807), `cmpeq` against the same + mask → 0x00/0xFF, `OR` with `mone` → `±1`, then `_mm256_sign_epi8(q8, ...)` + (x86:2871-2885). **Signs on the activation; grid stays unsigned.** Four vector ops per 32 + weights. +5. Scales: one 8-byte `memcpy`, nibble split and interleave, `scales = (stmp<<1) | 1` + (x86:2832-2835). Per-block broadcast via `_mm_shuffle_epi8(scales, get_scale_shuffle(ib32))` + + `cvtepi8_epi16` (x86:2892-2895), then `madd_epi16` (x86:2897-2900). +6. Two i32 accumulators, `fmadd`, `0.125f` at the end (x86:2903-2907). + +An SSE path mirrors all of this at 128 bits (x86:2909-3070). + +### (b) llama.cpp arm64, quants.c:3693-3763 + +1. **No parity arithmetic** — `signs64 + (q2[i] >> 9)` reads `keven_signs_q2xs` directly with + the 7-bit code (arm:3739-3742), and the table already carries the parity completion. The + x86 `pshufb` parity trick exists only because a 128-entry × 8-byte table read does not + vectorize on x86 the way `vld1_s8` does on NEON. +2. Grid: `vcombine_s8(vld1_s8(iq2xs_grid + (q2[0] & 511)), ...)` (arm:3735-3738). +3. Signs on the **weights**, `vmulq_s8` (arm:3743-3746). +4. Four `ggml_vdotq_s32` (arm:3747-3750). +5. **The best scale/reduction shape in the whole ARM file**: + `p = vpaddq_s32(vpaddq_s32(p1,p2), vpaddq_s32(p3,p4))` collapses four per-block i32x4 dot + vectors into one vector whose 4 lanes are the 4 block sums, then a single + `vmlaq_s32(sumi, p, scales32.val[ib64])` applies four per-block scales at once + (arm:3751-3752). Integer accumulation throughout; one `vaddvq_s32` per *superblock* + (arm:3755), not per block. Scales pre-expanded to `int32x4x4_t` outside the k-loop + (arm:3722-3731). + +### (c) What ours does today + +`emit_iq2xs_gather` (gemm_gen.das:1689-1745). Identical skeleton to IQ2_XXS: + +- The `uint16` word is assembled from **two separate `i8` loads plus a shift and an OR** + (gemm_gen.das:1711-1716). The offsets are `lp%4 ∈ {0,2}` within the same 4-byte column, so + a single 2-byte load would do. +- `ksigns` lookup from the 7-bit high field (gemm_gen.das:1718-1720) — the parity path, same + as NEON's, not x86's `pshufb` trick. Correct choice for a scalar decode. +- Grid: two `i32` loads from the `[1024 x i32]` low/high pair global + (gemm_gen.das:1578-1632, 1725-1727). +- Same 3-GPR-op negate and same 4-byte panel store (gemm_gen.das:1728-1732). +- Consumption: `iq2 = true` (gemm_gen.das:1817), so the lo and hi 16-weight halves keep + **split accumulators** `a[]`/`a1[]` and get separate per-16 unsigned scales + (gemm_gen.das:1893-1898, 1912-1919, 1932-1937) — the per-16 granularity IQ2_XS and IQ2_S + need. That part is right and is structurally what ARM's `vpaddq` + `vmlaq_s32` achieves, + without the reduction. + +### (d) Deltas, ranked + +Same 1/2/5/6/7 as IQ2_XXS, plus: + +| # | Delta | Candidate seat | Why | +|---|---|---|---| +| 3' | **One `i16` load for the packed word.** Replace the two `i8` loads + shift + OR (gemm_gen.das:1711-1716) with one aligned 2-byte load. | none — direct fix | 3 uops × 32 × mr per superblock, free. | +| 4' | **Split the sign byte into the plane at repack time.** IQ2_XS is the one format where the parity-completed byte does not fit for free (9 + 8 = 17 bits > 16). Costs +1 bit per 8 weights if adopted; race it against the `ksigns` load. | repack-side | Lower priority than for IQ2_XXS / IQ3_XXS where it is free. | +| 8 | **Race the `pshufb` parity form** as an alternative to the `ksigns` table read, once the sign path is vectorized (delta 2). llama.cpp measured it as *"a significant boost"* (x86:2822) on their layout. | `psign="parity"` | Only interesting if delta 2 lands first; on a scalar-index decode the table read wins. | + +--- + +## 5. IQ2_S + +Packing: 8-bit `qs` index + 2 bits from `qh` = 10-bit index into `iq2s_grid[1024]`; +**explicit sign bits** (`QK_K/8` bytes at `qs + QK_K/8`), no parity trick; nibble-packed +per-16 scales. + +### (a) llama.cpp x86 (AVX2), quants.c:3075-3160 + +1. Grid compose: `_mm256_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32] << 2) & 0x300)], ...)` + (x86:3123-3130) — index arithmetic done **scalar, in the address expression**, four + 64-bit reads per ymm. Note the shift amounts run `8,6,4,2` down the lanes so the same + `qh` byte feeds all four. +2. Sign expansion: `_mm256_set1_epi32(signs[0] | (signs[1] << 16))` broadcasts 4 sign bytes, + `shuffle_epi8(aux256, mask1)` spreads each byte across 8 lanes (`k_mask1`, x86:3090-3092), + `AND` with `k_mask2` = `{1,2,...,128}`×4 (x86:3094-3096), `cmpeq` → 0x00/0xFF + (x86:3133-3135). +3. Sign application: `_mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1)` (x86:3136) — a + **two-instruction conditional negate under a 0/-1 mask**, cheaper than + `or(mask,1)` + `psign` used by IQ2_XS, and applied **to the activation**. +4. `maddubs` (x86:3146-3147), then scale via + `_mm256_shuffle_epi8(scales16, get_scale_shuffle_k4(ib32))` + `madd_epi16` + (x86:3148-3149). Scales precomputed once per superblock as + `(nibble << 1) | 1`, `cvtepi8_epi16` (x86:3114-3116). +5. Two i32 accumulators, `fmadd`, `0.125f` at the end (x86:3155-3159). + +### (b) llama.cpp arm64, quants.c:3767-3859 + +1. Grid: four `vcombine_s8(vld1_s8(iq2s_grid + (qs[i] | ((qh[..] << k) & 0x300))), ...)` + (arm:3808-3816). +2. Sign expansion: `vdupq_n_u32(signs[0] | (signs[1]<<16))`, two `ggml_vqtbl1q_u8` with + `k_mask1` halves, `vandq_u8` with `k_mask2`, `vceqq_u8` (arm:3820-3825) — the direct + NEON analogue of the x86 shuffle+cmpeq. +3. Sign application: `vmulq_s8(vorrq_u8(vs, m1), q2s)` (arm:3826-3827) — `±1` on the + **weights**. +4. Four `ggml_vdotq_s32` (arm:3838-3841). +5. Scale: `sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32] & 0xf))` (arm:3843-3846) — + scalar integer accumulate, **four `vaddvq_s32` cross-lane reductions per 64 weights**. + Weaker than the IQ2_XS `vpaddq`+`vmlaq_s32` form on the same arch, for no obvious reason. + +### (c) What ours does today + +`emit_iq2s_gather` (gemm_gen.das:1518-1575). + +- Index: one `i8` load of `qs`, one `i8` load of the `qh` byte per block, `shl`/`and`/`or` + to build the 10-bit index, then `shl 1` for the low/high word pair + (gemm_gen.das:1543-1550). +- **Sign byte is read straight out of the plane** (gemm_gen.das:1541-1543) — no `ksigns` + indirection, because IQ2_S stores signs explicitly. This is the format where delta 2 + (vector sign expansion) is cheapest to reach: the sign bits are already plane-resident + and, given the `(p/4)*mr*4 + p%4` column layout (gemm_gen.das:1542), **one 32-byte load + covers 8 rows × 4 consecutive sign bytes at stride 4** — exactly the input a + `vpshufb`+`cmpeq` expansion wants. +- Same 3-GPR-op negate, same 4-byte panel store (gemm_gen.das:1557-1562). +- Consumption: the `iq2` split-accumulator per-16 path (gemm_gen.das:1817, 1893-1898). + +### (d) Deltas, ranked + +| # | Delta | Candidate seat | Why | +|---|---|---|---| +| 1 | Delete the panel (as §3). | `gather="reg"` | Same as every format. | +| 2* | **Vector sign expansion, promoted.** For IQ2_S specifically the 8 rows' sign bytes are one 32-byte strided column load away (gemm_gen.das:1542); `vpshufb` to select the wanted byte per row, `vpshufb`/`and`/`cmpeq` to expand to 32 mask bytes. ~5 vector ops replace 8 loads + 8 GPR negates. | `psign="expand"` | Highest-value sign delta of the five, because no `ksigns` table read stands in the way. | +| 3 | Adopt the **`xor`+`sub` conditional negate** (x86:3136) in place of `or(mask,1)`+`psign` wherever the mask is already 0/-1. | folded into `psign="expand"` | 2 ops vs 2 ops on x86, but frees `VPSIGNB` port pressure and works at 512 bits where `llvm.x86.avx2.psign.b` does not exist (see §9). | +| 5,6,7 | `vpdpbssd` seat, `bias=128` seat, `pmaddwd` scale fold — as §3. | | | + +--- + +## 6. IQ3_XXS + +Packing: 8-bit grid index into `iq3xxs_grid[256]` (4 weights each); a per-32 aux word with +four 7-bit sign codes + 4-bit scale, at `qs + QK_K/4`. + +### (a) llama.cpp x86 (AVX2), quants.c:3260-3316 + +1. Grid compose: `_mm256_set_epi32(iq3xxs_grid[q3[7]], ..., iq3xxs_grid[q3[0]])` + (x86:3290-3295) — **eight 32-bit table reads written directly into one ymm**, twice per + loop iteration. This is the exact composition our layout also needs. +2. `memcpy(aux32, gas, 8)` staging (x86:3296). +3. Signs: four `signs64[(aux32[k] >> 7j) & 127]` per ymm (x86:3297-3300), `keven` table, + parity already baked. +4. `_mm256_sign_epi8` on the activation (x86:3301-3302); `maddubs` (x86:3303-3304). +5. Scale `2*ls+1` via `set1_epi16` + `madd_epi16` (x86:3307-3308); two i32 accumulators; + `fmadd`; `*s = 0.25f * hsum_float_8` (x86:3316). + +### (b) llama.cpp arm64, quants.c:3864-3922 + +1. Grid compose: `ggml_vld1q_u32(iq3xxs_grid[q3[0]], ..., q3[3])` (arm:3896-3899) — the macro + (ggml-cpu-impl.h:82-84) is a brace-initialised `uint32x4_t`, i.e. four scalar dword reads + the compiler materialises with `ld1 {v.s}[lane]` or `ins`. Same shape as `set_epi32`. +2. Signs from `keven_signs_q2xs`, `vcombine_s8` of two 8-byte reads (arm:3901-3904). +3. `vmulq_s8` on the **weights** (arm:3905-3908). +4. Chained `ggml_vdotq_s32` ×2 per 32 weights (arm:3909-3910). +5. `sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[0] >> 28))` (arm:3911-3912) — again the + per-block cross-lane reduction and **float** accumulate; `*s = 0.5f * sumf` (arm:3916). + +### (c) What ours does today + +`emit_iq3xxs_gather` (gemm_gen.das:1418-1474). Cleanest of our five gathers: + +- **One aligned `i32` load** for the aux word (gemm_gen.das:1438-1439) — the spelling + IQ2_XXS should copy. +- `ksigns` lookup per sub-lane (gemm_gen.das:1442-1444). +- Two `i8` index loads (gemm_gen.das:1445-1450); grid is the **halved** `[256 x i32]` global + (gemm_gen.das:1362-1377, 1455-1457). +- Same 3-GPR-op negate + 4-byte panel store (gemm_gen.das:1458-1462). +- Consumption: `iq2 = false` (gemm_gen.das:1817), so lo and hi share one accumulator and one + per-32 signed scale (gemm_gen.das:1899-1901, 1920-1921) — matching the per-32 granularity. + +### (d) Deltas, ranked + +1. **Delete the panel** (`gather="reg"`) — the single biggest item, and the composition it + needs is literally x86:3290's `set_epi32`, transposed from "8 grid words of one row" to + "one grid word from each of 8 rows". +2. **Vector sign apply** (`psign="mask"` / folded into delta 1) — deletes 24 GPR ops and + `VPABSB` per 32-byte vector. +3. **Bake the parity-completed sign byte at repack time.** Aux is 4×7 sign bits + a 4-bit + scale, and the scale already lives on the scale row — so four full sign bytes fit in the + same 4 bytes, exactly as for IQ2_XXS. Free; deletes the `ksigns` load. +4. `vpdpbssd` seat; `bias=128` seat (needs the `te.bias` clamp lifted, gemm_gen.das:2136); + `pmaddwd` scale fold — as §3. + +--- + +## 7. IQ3_S + +Packing: 8-bit `qs` + one bit from `qh` = 9-bit index into `iq3s_grid[512]`; **explicit +sign bytes** (`x[i].signs`); nibble-packed per-32 scales. + +### (a) llama.cpp x86 (AVX2), quants.c:3384-3475 + +1. **Vectorized index compose.** `idx_l = _mm256_cvtepu8_epi16(load 16 qs bytes)`; + `idx.vec[j] = _mm256_and_si256(_mm256_sllv_epi32(set1_epi32(qh[..]), idx_shift), idx_mask)` + with `idx_shift = {1..8}` and `idx_mask = 256`, then `OR` with the widened `qs` + (x86:3430-3438). Eight 9-bit indices built in one ymm with a variable shift. +2. **The index goes through memory, in the safe direction.** `index_t` is a union of + `__m256i vec[2]` and `uint32_t index[16]` (x86:3406-3411); the vector is stored and the + dwords are read back scalar to address the grid (x86:3441-3450). One wide store feeding + 16 narrow, fully-contained loads forwards fine — the opposite of our panel's one wide + load over 8 narrow stores. +3. **An explicit negative result on hardware gather**, x86:3440: + `// At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than + _mm256_set_epi32. Strange.` — with the `vpgatherdd` version left commented out at + x86:3441-3442. Do not spend time on a gather-instruction spelling. +4. Sign expansion: identical to IQ2_S — `set1_epi32(signs[0] | signs[1]<<16)`, + `shuffle_epi8(mask1)`, `and(mask2)`, `cmpeq` (x86:3452-3459). +5. Sign application: `sub(xor(s2, q8), s2)` (x86:3455, 3460) — **on the activation**. +6. `maddubs` (x86:3464-3465); scale `2*ls+1` from the nibble pair via `set1_epi16` + + `madd_epi16` (x86:3466-3469); two i32 accumulators; `fmadd`; **no global factor** — + `*s = hsum_float_8(accumf)` (x86:3475). + +### (b) llama.cpp arm64, quants.c:3926-4032 + +1. Index compose in vector: `vorrq_u16(vmovl_u8(qs_half), vandq_u16(vshlq_u16(vdupq_n_u16(qh), + hshift), m256))` with `k_shift = {8,7,...,1}` (arm:3985-3986, 3992). +2. Same union round trip — `vec_index_t { uint16x8_t vec_index; uint16_t index[8]; }` + (arm:3944-3947), stored then read back scalar for `ggml_vld1q_u32(iq3s_grid[idx.index[0]], + ...)` (arm:3987-3996). +3. Sign expansion via `vqtbl1q_u8` + `vceqq_u8` + `vorrq_u8(..., m1)` → `±1` + (arm:3998-4012); applied to the **weights** with `vmulq_s8` (arm:4004-4005, 4016-4017). +4. **Scales precomputed with a SWAR trick**: `memcpy(scales32, x[i].scales, 4)`, then + `scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101` and the twin for the + low nibbles (arm:3976-3979) — eight `(2s+1)` scale bytes computed with four GPR ops per + superblock. +5. `sumi1 += vaddvq_s32(p1) * scales8[ib32/2]` (arm:4020-4023) — per-block cross-lane + reduction, scalar integer accumulate; `*s = sumf` (arm:4028). + +### (c) What ours does today + +`emit_iq3s_gather` (gemm_gen.das:1300-1359), the arm the followup entry describes. + +- `qh` byte load per block (gemm_gen.das:1319-1321), sign byte load per sub-lane + (gemm_gen.das:1323-1325), `qs` byte load + `shl`/`and`/`or` per grid word + (gemm_gen.das:1327-1333). **All scalar, one byte at a time** — no `sllv`-style vector index + compose. +- Grid: `[512 x i32]` private constant, `align 64` (gemm_gen.das:1260-1272), one `i32` load + per 4 weights (gemm_gen.das:1339-1341). +- Sign: 16-entry nibble→byte-mask table (`dasllama.iq3s.smask`, gemm_gen.das:1274-1293), then + `(gw ^ mw) + (mw & 0x01010101)` (gemm_gen.das:1342-1344). +- 4-byte panel store (gemm_gen.das:1345-1347). +- Consumption: shared per-32 signed scale path (gemm_gen.das:1899-1901, 1920-1921); scale + applied as `vpmulld` on the i32 accumulator; one `sitofp` + FMA per superblock; global + factor pre-baked into the scale row. + +### (d) Deltas, ranked + +| # | Delta | Candidate seat | Why | +|---|---|---|---| +| 1 | **Delete the panel** — compose 8 grid dwords per weight vector with `insertelement`. | `tune_perm(dot="maddubs", width=256, mr=8, gather="reg", requires="avx2")` | The 0.92x tail's stated cause, and the measurement in followup 61 supports it. | +| 2 | **Vector sign expansion off the plane.** Like IQ2_S, the sign bytes are explicit and plane-resident at stride 4 across rows (gemm_gen.das:1324) — one 32-byte column load + `vpshufb`/`and`/`cmpeq` gives the full 32-byte `±1`/mask vector. | `psign="expand"` | Deletes the `smask` table read *and* the 3-GPR-op negate *and* `llvm.abs`. | +| 3 | **Vectorized index compose** (`sllv` shape, x86:3430-3438 / arm:3985-3986) feeding a *wide-store / narrow-load* union — the safe memory direction — instead of per-byte `qs` loads. | `idx="vec"` | Only worth racing after delta 1; the win is ~4 GPR ops per grid word. Note llama.cpp deliberately keeps the *grid read* scalar even here. | +| 4 | Do **not** race `vpgatherdd` for the grid read. | — | x86:3440 records it losing to `set_epi32` on Zen 4. | +| 5 | Adopt the ARM SWAR scale expansion (arm:3976-3979) if the scale row is ever computed at runtime. | — | Ours pre-bakes scales in the repack, so this is already better; noted for completeness. | +| 6,7 | `vpdpbssd` seat; `bias=128` seat; `pmaddwd` scale fold — as §3. | | | + +--- + +## 8. Verdict on `modules/dasLLAMA/followup_general.md` entry 61 + +Entry 61 says: + +> The stamped gemv gathers each superblock into an alloca panel and then runs the vector dot +> — a store/load round trip per superblock that a single token never amortizes; llama.cpp's +> per-row form (grid words composed straight into vectors, signs applied to the ACTIVATION +> via shuffle+cmpeq/xor-sub, magnitudes kept unsigned for maddubs) edges it 57.0 vs 52.4 +> tg128 on the zen2. + +**Confirmed, for all five formats, not just IQ3_S:** + +- The panel round trip is real and shared. Every one of `emit_iq3s_gather`, + `emit_iq3xxs_gather`, `emit_iq2s_gather`, `emit_iq2xs_gather`, `emit_iq2xxs_gather` ends + each decoded dword with `LLVMBuildStore(b, w, dp)` into `te.iq3s_panel` + (gemm_gen.das:1345-1347, 1460-1462, 1560-1562, 1730-1732, 1792-1794), and + `emit_block_iq4xs` reloads it as `` (gemm_gen.das:1880-1883). The panel is a + GEMV-only cost: in tile mode `te.iq3s_panel` is null (`kq_tile_gen_impl` never sets it, + gemm_gen.das:3200-3244) and the byte-expanded panel is built once per row-group per + token-block by `unpack_kq_panel_grp` (math_gen.das:2091), amortized over up to TB tokens. + That is exactly why prefill wins and decode loses. +- llama.cpp does compose grid words straight into vectors (x86:2691, 2856, 3123, 3290, 3444; + arm:3664, 3735, 3808, 3896, 3987) and never materializes a decoded-weight buffer. +- Magnitudes-stay-unsigned-for-`maddubs` is right on x86 (grid is the u8 operand at + x86:2699, 2887, 3146, 3303, 3464). + +**One clause needs correcting:** + +- *"signs applied to the ACTIVATION"* is **not portable to our layout.** It is true of + llama.cpp's x86 kernels only, and it is a consequence of their one-row-at-a-time shape. + In our `mr`-interleaved GEMV the lanes of one weight vector are 8 different output rows + with 8 different sign patterns, while the activation dword is splatted across all lanes — + so a single `psign` on the activation cannot be correct. llama.cpp's own **arm64** kernels + make the same concession for the same reason (`sdot` is signed×signed): they apply signs to + the **weights** with `vmulq_s8` at arm:3672, 3743, 3826, 3905, 4004. Adopting + "signs-on-activation" would mean abandoning row interleave (`mr = 1`), which would give + back the per-row horizontal reduction we currently avoid entirely. Not worth it. +- What *is* portable, and is the real content of the delta, is: **keep the grid magnitudes + unsigned and build the sign mask as a vector**, so `psign(x_splat, mask|1)` replaces + `abs(panel_bytes)` + `psign(x_splat, panel_bytes)` and the 3-GPR-op-per-dword negate + disappears with the panel. The `abs`+`psign` pair is dropped outright only by + `vpdpbssd` (signed×signed, gemm_gen.das:344-347) or by `bias=128` + (gemm_gen.das:339-343) — neither of which is seated for any grid format today. +- Entry 61 is also **under-scoped**: it is filed as an IQ3_S item, but the panel, the gather + shape, the scalar sign negate and the `abs`+`psign` pair are byte-for-byte identical in all + five gathers. One `gather="reg"` spelling in `emit_block_iq4xs` covers all five at once. + +--- + +## 9. Cross-format section — what one spelling would cover + +All five gathers are the same function with a different index/sign preamble. Ranked by +estimated decode-shape payoff: + +### Tier 1 — one change, all five formats + +**`gather="reg"`: compose the weight vector in registers, delete the panel.** + +Where: `emit_block_iq4xs` (gemm_gen.das:1811-1957) and the five gather emitters. Restructure +the gather loop from *row-major* (`for r in mr { for blk, l, h }`, gemm_gen.das:1305-1352) +to *group-major* (`for blk, j, half { for r in mr }`), emitting `insertelement` of each row's +decoded dword into an `` and bitcasting to `te.vwi8` — the same value +`load_vec(te, te.vwi8, wbase, noff)` returns today (gemm_gen.das:1880-1883), produced without +touching memory. + +Why it is the top item: + +- Removes `256 × mr / 4 = 512` scattered 4-byte stores per superblock (mr = 8) and the + 64 wide reloads, plus the `alloca` traffic. +- Removes the **wrong-direction store-forwarding hazard**: the reload at + `(blk*16 + j*4)*mr` (gemm_gen.das:1875) is a 32-byte load spanning 8 distinct 4-byte + stores written `mr*4 = 32` bytes apart (gemm_gen.das:1345). x86 forwards a narrow load out + of a wide store; it does not forward a wide load out of several narrow stores. llama.cpp's + own memory round trips (x86:2823 `aux_gindex`, x86:3406 `index_t`, arm:3944 + `vec_index_t`) are all in the *safe* direction and carry indices, never decoded weights. +- Worse at width 512 today, better after: the `mr = 16` seat writes 1024 stores into a + 4096-byte panel and reloads with 64-byte loads spanning 16 stores each. +- Rough per-32-byte-vector uop budget: ~51 now → ~20 composed. + +Seats (one per existing perm row, five formats × four rows): + +``` +tune_perm(mr = 8, gather = "reg"), +tune_perm(dot = "maddubs", width = 256, mr = 8, gather = "reg", requires = "avx2"), +tune_perm(dot = "vpdpbusd", width = 256, mr = 8, gather = "reg", requires = "avxvnni|avx512vnni"), +tune_perm(dot = "vpdpbusd", width = 512, mr = 16, gather = "reg", requires = "avx512vnni,avx512bw"), +``` + +(Suffix derivation: `suffix_piece` in llvm_tune.das:1317-1323 renders a string arg as +`name_value`, so these stamp as `..._gather_reg`; add them to each `tune(... fallback = ...)` +chain in priority order.) + +### Tier 2 — one change, all five formats + +**`psign="mask"` / `psign="expand"`: build the sign vector, stop negating in GPRs.** + +Composed with tier 1: keep the raw grid dword as the unsigned magnitude (it already is — +ggml-common.h:561, 1018, 1053), compose a second vector of 0x00/0xFF masks, and apply +`vpor(mask, 1)` + `VPSIGNB` to the splatted activation — or the cheaper +`sub(xor(mask, x), mask)` conditional negate llama.cpp uses at x86:3136/3455. Deletes: + +- the three GPR ops per dword at gemm_gen.das:1342-1344 / 1458-1460 / 1557-1559 / + 1728-1730 / 1790-1792 (~24 ops per 32-byte vector), and +- `abs_w` / `llvm.abs` / `VPABSB` from the inner loop (gemm_gen.das:370-375, 1888-1889). + +Two sub-variants worth racing separately: + +- `psign="mask"` — compose the mask vector from the existing 16-entry `smask` global with + `insertelement` (8 more folded loads). +- `psign="expand"` — read the sign bits vectorized off the plane and expand with + `vpshufb`+`and`+`cmpeq` (llama.cpp's `k_mask1`/`k_mask2`, x86:3090-3096 / arm:3782-3786). + Cheapest for **IQ2_S** and **IQ3_S**, whose sign bytes are plane-resident at stride 4 + across rows (gemm_gen.das:1542, 1324); needs one `ksigns` scalar read per 8 weights for + IQ2_XXS/IQ2_XS/IQ3_XXS unless the repack change below lands. + +Note `llvm.x86.avx2.psign.b` is only wired at `width == 256` (gemm_gen.das:2185-2190); the +512-bit leg already falls back to a `select` (gemm_gen.das:358-362). The `xor`+`sub` form +works at both widths and would simplify that split. + +### Tier 3 — seats only, zero generator change + +**`dot = "vpdpbssd"` for all five formats.** The panel/composed bytes are signed and the +activation is signed, so `dot_lane` takes the native `s8·s8` path with no `abs` and no +`psign` (gemm_gen.das:344-347, 370-372). `perm_declines` already permits it at width 256 +under `avxvnniint8` (gemm_gen.das:212-220). One line per format: + +``` +tune_perm(dot = "vpdpbssd", width = 256, mr = 8, requires = "avxvnniint8"), +``` + +Only helps AVX-VNNI-INT8 hosts (not zen2, not zen4), so file it as cheap coverage, not as +the fix for the measured tail. + +### Tier 4 — repack-side, three of five formats + +**Bake the parity-completed sign byte into the plane.** `ksigns_iq2xs` maps a 7-bit code to +the 8-bit sign byte (ggml-common.h:513). For **IQ2_XXS** and **IQ3_XXS** the aux word is +4×7 sign bits + a 4-bit scale, and the scale nibble already moved to the scale row +(gemm_gen.das:1746-1748) — so four full sign *bytes* occupy the same four bytes, at zero +size cost, and the `ksigns` load disappears (gemm_gen.das:1442-1444, 1775-1776). For +**IQ2_XS** it costs +1 bit per 8 weights (9 index bits + 8 sign bits > 16), so race it. This +also promotes IQ2_XXS/IQ3_XXS into the cheap `psign="expand"` class alongside IQ2_S/IQ3_S. + +### Tier 5 — small, local, do anyway + +- **IQ2_XXS aux word**: four `i8` loads + 3 shifts + 3 ORs (gemm_gen.das:1766-1770) where + IQ3_XXS uses one aligned `i32` load of the identically-aligned field + (gemm_gen.das:1438-1439). 6 uops × 8 blocks × mr rows per superblock, free to reclaim. +- **IQ2_XS packed word**: two `i8` loads + shift + OR (gemm_gen.das:1711-1716) where one + 2-byte load suffices (`lp % 4 ∈ {0,2}`, so the pair never straddles the 4-byte column). +- **Panel/vector load alignment**: `load_vec` hard-codes `align 1` + (gemm_gen.das:376-379) although the alloca is `align 32` (gemm_gen.das:3082) and every + offset is a multiple of `mr*4`. Moot once the panel is gone, but it also affects the tile's + reads of the unpacked panel. +- **`gkstep` is inert for these formats**: `kq_gemv_gen_impl` pins `SliceArgs(kstep = 1)` + (gemm_gen.das:3084) and never reads `p.gkstep`, while `perm_declines` still validates it + (gemm_gen.das:169). Any `gkstep` seat added to a grid format would be a no-op clone. +- **`bias = 128` is clamped off** for every `kq` format at gemm_gen.das:2136 + (`te.bias = (te.mx4 || te.q51 || te.kq != 0) ? 0 : p.bias`). Since we synthesize the panel + bytes ourselves, `w ^ 0x80808080` is a free per-byte `+128` (no carries), so lifting the + clamp for the five panel formats would make the `bias128` lattice — which drops `abs` and + `psign` entirely and folds the `-128·Σx` correction into the accumulator init + (`bias_acc_init`, gemm_gen.das:400-427) — available on VNNI hosts. **Not** valid under + `maddubs`: `(w+128) × x` pair sums reach 65280 and saturate `i16`, which is why the knob is + documented `vpdpbusd`-only (gemm_gen.das:44-47, 170-171). + +### What we already do better, and should not regress + +- **No horizontal reduction anywhere.** llama.cpp's arm64 kernels pay 4-8 `vaddvq_s32` per + superblock per row (arm:3678, 3752, 3845, 3911, 4021); IQ2_XXS and IQ3_XXS additionally + accumulate in **float** per 32-weight block (arm:3678-3679, 3911-3912). Our lanes are rows, + so the superblock accumulates in `i32` and converts once (gemm_gen.das:1943-1953). +- **Global factor pre-baked.** llama.cpp multiplies by `0.125f` / `0.25f` / `0.5f` per row at + the end (x86:2713, 3161, 3316; arm:3684, 3760, 3916). Ours rides the repacked scale row. +- **Per-16 split accumulators for IQ2_S/IQ2_XS** (gemm_gen.das:1817, 1893-1898) match the + formats' actual scale granularity without any extra reduction — structurally what + llama.cpp's best ARM arm (IQ2_XS `vpaddq`+`vmlaq_s32`, arm:3751-3752) achieves, and better + than what its IQ2_S arm does (scalar `vaddvq` × scalar scale, arm:3843-3846). +- **Q2_K needs no work.** Our `k2` arm decodes 2-bit lanes in-register with a shift and an + `and` (gemm_gen.das:1049-1056) — the same technique as llama.cpp AVX2 (x86:1626-1629) and + NEON (arm:1988-1996) — with no panel, no gather, no table. That is exactly why it measures + at parity, and it is the control that proves the gap in the other five is the *decode*, + not the dot or the fold. diff --git a/plans/kernel_parity_research_vk.md b/plans/kernel_parity_research_vk.md new file mode 100644 index 0000000000..ae8b68c209 --- /dev/null +++ b/plans/kernel_parity_research_vk.md @@ -0,0 +1,470 @@ +# Technique memo: llama.cpp's cooperative-matrix-2 prefill GEMM vs ours + +Read-only research. Trees: `D:\Work\llama.cpp` at `6c84c7d5d` (2026-08-27) and +`D:\Work\daScript-iquant`. Nothing in either tree was modified. + +Target box for every number below: RTX 5060 Ti (Blackwell GB206, 36 SMs), the box +`D:\Work\daScript-iquant\modules\dasLLAMA\ARCHITECTURE_GPU_VULKAN.md:49` names. + +One environment fact that shapes the whole comparison: the local llama.cpp Vulkan build +reports `GL_NV_cooperative_matrix_decode_vector not supported by glslc` +(`C:\Users\Boris\.claude\jobs\7f9893d9\tmp\tbo_build_vk.log`). llama.cpp therefore runs the +scalar (one element per callback) decode path here, the same as ours. The comparison is +apples to apples today. It stops being so the moment that glslc is upgraded — see delta 7. + +--- + +## 1. llama.cpp's cm2 GEMM, in ten facts + +**F1 — One shader, six pipelines per quant type, three tile geometries.** +Geometry arrives as spec constants `{BLOCK_SIZE, BM, BN, BK, enable_smaller_matrices}` +(`D:\Work\llama.cpp\ggml\src\ggml-vulkan\vulkan-shaders\mul_mm_cm2.comp:31-39`). On coopmat2 the +values are large `{256, 128, 256, 64, 1}`, medium `{256, 128, 128, 64, 1}`, small +`{256, 32, 64, 128, 0}` — and they are byte-identical for legacy quants and for K-quants +(`D:\Work\llama.cpp\ggml\src\ggml-vulkan\ggml-vulkan.cpp:4263-4277`). Each is stamped twice, +unaligned and aligned, giving six (`ggml-vulkan.cpp:4664-4671`). There is **no per-format tile +tuning** on the cm2 path. + +**F2 — Operands: f16 times f16 into an f16 accumulator, stored as f32.** +`mat_a` is `BM x BK` of the quantized weight, `mat_b` is `BK x BN` of the activation, both +`MAT_TYPE = FLOAT_TYPE = float16_t` on coopmat2 (`mul_mm_cm2.comp:98-102`, +`vulkan-shaders-gen.cpp:492-529`). `coopMatMulAdd` accumulates into `ACC_TYPE`, which is +`float16_t` at `GGML_PREC_DEFAULT` and `float` only at `GGML_PREC_F32` +(`vulkan-shaders-gen.cpp:471-475`; the pick is `ggml-vulkan.cpp:7818-7820`). The device is only +admitted to the cm2 path if it advertises fp16 A and B with both fp16 and fp32 accumulators at +workgroup scope and flexible dimensions (`ggml-vulkan.cpp:6806-6838, 6863-6869`). Result is +converted to `D_TYPE = float` for the store (`mul_mm_cm2.comp:493-495`). + +**F3 — There is no integer activation path on coopmat2.** +An f32 `src1` on a coopmat2 device is unconditionally marked non-contiguous +(`ggml-vulkan.cpp:9276-9278`), which forces `quantize_y` false (`ggml-vulkan.cpp:9285`), and the +q8_1 integer-dot `mul_mmq.comp` shader is never even compiled for coopmat2 +(`vulkan-shaders-gen.cpp:626`). Activations are always f16 here. + +**F4 — The f32-to-f16 activation conversion is one dispatch, cached across sibling matmuls.** +It runs through `ggml_vk_cpy_to_contiguous` into the `prealloc_y` scratch, but only when the +last conversion did not already produce the same tensor with the same pipeline +(`ggml-vulkan.cpp:9441-9453`, guarded by `prealloc_y_last_pipeline_used` / +`prealloc_y_last_tensor_used`). Q, K and V therefore pay one conversion between them. + +**F5 — Weights are decoded inside the driver's block load; nothing is pre-dequantized.** +`coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(...), dequantFuncA)` +(`mul_mm_cm2.comp:381, 468`). The tensor layout carries the quant block size +(`setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K)`, `mul_mm_cm2.comp:315-318`) so the +driver resolves the block pointer and hands the callback a `buffer_reference` to it. The weight +stays in ggml's on-disk block layout: no repack, no staging pass, no extra VRAM. + +**F6 — Almost nothing is staged in shared memory, and the one exception is Q4_K/Q5_K scales.** +`mul_mm_cm2.comp` declares only `row_ids` and `ballots_sh`, both under `MUL_MAT_ID` +(`mul_mm_cm2.comp:108, 116`) — contrast the non-cm2 shader, which stages both operands +(`mul_mm.comp:130-131`). The exceptions are the iq codebooks +(`types.glsl:918, 1031, 1201, 1501, 1573, 1683, 1813`, copied by `init_iq_shmem`) and a +**Q4_K/Q5_K-only scale hoist**: `fetch_scalesQ4_K` reads the whole 16-byte scale head of a +superblock as ONE `uvec4` (`dequant_funcs_cm2.glsl:449, 461`, via the +`buffer_reference_align = 16` `block_q4_K_packed128` alias at `:429-431`), and +`store_scalesQ4_K` unpacks it into 8 ready-multiplied `(d, m)` pairs in +`shAscales[8 * (BM+2)]` (`:442-444, 484-523`). Every element decode then reads one `vec2` with +no shifting at all (`:537-540`). The comment states the intent outright: "we decode a tile's +worth of scales into shared memory and then process the whole tile using those scales" +(`:435-440`). + +**F7 — Two tensor loads, one multiply-add, zero barriers per k-step.** +The inner body is exactly `coopMatLoadTensorNV(mat_a, ...)`, `coopMatLoadTensorNV(mat_b, ...)`, +`sum = coopMatMulAdd(mat_a, mat_b, sum)` (`mul_mm_cm2.comp:464-473`). The only barriers in the +k loop are the two inside `store_scalesQ4_K` (`dequant_funcs_cm2.glsl:486, 522`), which fire once +per `QUANT_K = 256` elements of k — that is once per four `BK = 64` steps — and only for +Q4_K/Q5_K. Every other format runs the whole k loop barrier-free. The scale fetch and store are +explicitly pipelined: the next block's scales are fetched while the current block's are being +consumed (`mul_mm_cm2.comp:363-374`). + +**F8 — A clamp-free fast path, entered by construction rather than by luck.** +The fast path requires a whole M tile, a whole N tile against `padded_N`, `start_k % 256 == 0`, +`end_k % BK == 0` and `stride_b % 8 == 0` (`mul_mm_cm2.comp:344-349`). Strides are then masked +`&= ~7` purely to make 16-byte alignment provable to the driver's address analysis +(`:299-306, 350-355`). Crucially, the N side is **padded in the buffer**: +`padded_n = ROUNDUP_POW2(ne11, pipeline->wg_denoms[1])` (`ggml-vulkan.cpp:9319`), so even the last +token tile takes the unclamped path. The unaligned fallback uses clamp-Constant layouts and is +marked `[[dont_unroll]]` (`mul_mm_cm2.comp:607`). + +**F9 — The tail N tile gets a narrower accumulator.** +`enable_smaller_matrices` makes the final column tile allocate `BN/2` or `BN/4` accumulators and +loop over that width (`mul_mm_cm2.comp:36-38, 367-409, 410-452`), so a partial token column does +not carry a full-width accumulator through the whole k loop. + +**F10 — Split-k with a separate vectorised reduce.** +Split fires when `k >= 2048` and the tile grid fills less than half the SMs; the factor is +`shader_core_count / tiles`, capped at 8, with the chunk rounded to 256 and splits shed if the +last chunk would be empty (`ggml-vulkan.cpp:8795-8826`). Partials land in `prealloc_split_k` and a +256-thread `vec4` shader sums them (`ggml-vulkan.cpp:8909-8925`, +`mul_mat_split_k_reduce.comp:17-47`). + +**Worked pick for a 1B prefill shape on this box** (M = 2048 weight rows, N = 512 tokens, +K = 2048, 36 SMs): `tiles_m = 16 * 4 = 64 > 36`, so `prefer_large` is true and the aligned large +pipeline is chosen — BM 128, BN 256, BK 64, 256 invocations +(`ggml-vulkan.cpp:8839-8863`, `9309-9312`). Split-k: 32 tiles, neither `<= 18` nor `<= 24`, so +`split_k = 1` (`ggml-vulkan.cpp:8795-8807`). Grid is 16 by 2 by 1 workgroups +(`ggml-vulkan.cpp:8896`, `8221-8229`). + +Batch and MoE, for completeness: plain batches ride `gl_WorkGroupID.z` with `broadcast2`/ +`broadcast3` (`mul_mm_cm2.comp:253-261`); MoE builds a shared `row_ids` table with a +subgroup-ballot prefix scan and stores through a per-element callback +(`mul_mm_cm2.comp:163-227, 150-161`). + +--- + +## 2. Ours, in the same terms + +**O1 — Same three tile geometries, same workgroup size, same accumulator type.** +`KqCm2BatchT::run` is `[spirv_kernel(local_size_x = 256)]` +(`D:\Work\daScript-iquant\modules\dasLLAMA\dasllama\dasllama_vulkan_classes.das:4825`). A is +`coopmatWgA_f16_128x64` (`:4888`), so BM 128 and BK 64 are fixed; BN is a template constant of +256, 128 or 32 with matching B and accumulator types (`:4809`, `:5336-5361` for q8 and the same +four typedefs repeated per format through `:5713`). The accumulator is +`coopmatWgAcc_f16_128xBN` — f16, like llama.cpp's default — clamped to the f16 range and widened +to f32 only for the store (`:4923-4926`). + +**O2 — Same tile pick, arrived at independently and landing in the same place.** +`cm2_tile_cols` compares occupied against allocated wave slots cross-multiplied, ties to large, +with `cnt <= 64` forcing small and `cnt <= 128` forcing medium +(`D:\Work\daScript-iquant\modules\dasLLAMA\dasllama\dasllama_vulkan_common.das:129-151`). For +d = 2048, cnt = 512, 36 cores it returns 256, the same large tile llama.cpp picks. + +**O3 — Same split-k rule, near-identical constants.** +`cm2_split_k` gates on `n >= 2048`, `d >= 128`, `cnt >= tc`, computes `cores / tiles` capped at +8, rounds the chunk to 256 and sheds stranded splits +(`dasllama_vulkan_common.das:155-179`). The in-kernel plane decomposition is +`dasllama_vulkan_classes.das:4832-4846`; the reduce is a separate class +(`:5722-5751`), encoded at `dasllama_vulkan_prefill.das:355-366`. + +**O4 — Same decode-in-load mechanism.** +`coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode)` +(`dasllama_vulkan_classes.das:4911`), with the block size on the layout +(`tensorLayoutSetBlockSize(fla, 1u, BLKW)`, `:4896`). Weights are never materialised in f16. + +**O5 — Same fast/edge split and the same stride-masking trick.** +Fast path when `m0 + 128 <= pa.d`, the token column is whole or the small tile is stamped, and +`pa.n % 64 == 0` (`:4893`); strides masked `& ~7u` on the B and output layouts (`:4902, 4906`). +Edge path uses clamp-Constant layouts (`:4950-5005`). + +**O6 — Same k-loop shape, one degree more unrolling, zero barriers.** +Two loads and one `coopmatMulAdd` per 64-wide k step, the outer loop unrolled 8 deep for 512 k +per iteration with a scalar drain (`:4907-4922`). The kernel contains exactly one `barrier()`, +before the loop, to publish `wg_blk0` and any staged codebook (`:4887`). llama.cpp unrolls 4 deep +at `QUANT_K = 256` (`mul_mm_cm2.comp:341-342`). + +**O7 — The activation f16 conversion is likewise one dispatch per sibling group.** +`f16cvt_cls` feeds q/k/v (`dasllama_vulkan_prefill.das:518-524`) and gate/up (`:562-568`); +`actf16_cls` fuses the SwiGLU epilogue with the conversion feeding down (`:569-576`). All of it +is recorded into the measured command buffer. This is at least as good as llama.cpp's caching, +and the fused activation is strictly better. + +**O8 — Difference: the weight scale plane is a second SSBO, addressed by hand, per element.** +`wq` (quants) is binding 0 and `ws` (scales) is binding 1 +(`dasllama_vulkan_classes.das:4804-4805`). Every decode callback computes its own scale address +`(wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u` and then extracts the sub-block scale with a 32-bit +load and a **runtime-variable shift** — per decoded element. Q4_K is the worst case, three +separate `ws` dwords at a 20-byte stride plus three variable shifts +(`:5034-5038`). Q6_K (`:5058-5062`), Q3_K (`:5143-5147`), Q2_K (`:5164-5167`), IQ4_XS +(`:5122-5125`), IQ3_S (`:5190-5193`), IQ3_XXS (`:5223-5226`), IQ2_S (`:5269-5273`) and IQ2_XS +(`:5295-5299`) all carry the same shape. llama.cpp reads every scale from the same block the +driver already resolved, and for Q4_K/Q5_K does the extraction eight times per 256 elements +instead of 256 times (F6). + +**O9 — Difference: no shared-memory scale hoist for any format.** +The only workgroup memory declared is `wg_blk0` (4 bytes) plus one gated iq codebook of 32 bytes +to 8 KB (`dasllama_vulkan_classes.das:4808-4823`). There is no `shAscales` equivalent. + +**O10 — Difference: the codebooks are computed per workgroup, not copied.** +`iq3s_gridc[gi] = iq3s_grid_word(int(gi))` and its siblings synthesise the table with arithmetic +at kernel entry (`:4861-4886`), where llama.cpp copies from a constant array +(`types.glsl:1204-1213`). + +**O11 — Difference: a per-workgroup prologue that llama.cpp does not have.** +Each workgroup reads a region record from a `sched` SSBO — five scalar loads +(`dasllama_vulkan_classes.das:3972-3976`) — broadcasts `wg_blk0` through workgroup memory +(`:4849-4851`) and barriers (`:4887`). llama.cpp derives everything from `gl_WorkGroupID` with no +loads and no leading barrier. Ours buys MoE region lists with this; llama.cpp pays for the same +capability with a separate `MUL_MAT_ID` shader. + +**O12 — Difference: no N padding and no narrow-tail accumulator.** +The activation plane is sized to `cnt`, not to a tile multiple, for the large and medium tiles +(`:4901`), so a partial last token column drops to the fully clamped edge path (`:4893`). Only +the small tile rounds its dimension up to 32 and keeps the unclamped load. There is no +`enable_smaller_matrices` analogue. At exactly 512 tokens with BN 256 this costs nothing; at +500 tokens it costs the last tile. + +**O13 — Grid: one-dimensional.** +`vkCmdDispatch(raw, groups, 1u, 1u)` (`dasllama_vulkan_common.das:2469-2478`) with +`groups = ceil(rows/BN) * ceil(d/128) * nsplit` (`dasllama_vulkan_prefill.das:1089-1106`). Push +constants are 16 bytes, `{n, d, map_off, ksplit}` (`dasllama_vulkan_classes.das:2326-2331`); +everything else rides the SSBO region record. llama.cpp pushes a 17-word block with the full +geometry (`mul_mm_cm2.comp:41-70`). + +**O14 — The measured ratios are NOT a uniform band, and the followup says otherwise.** +`followup_vulkan.md:540-546` (entry 34) states "~0.67-0.70x" and "every sb-format cm2 tile lands +in the same band" and "the gap is the shared batch-GEMM tier, not any one format's decode". +The per-format tables in +`D:\Work\daScript-iquant\modules\dasLLAMA\HOW_TO_ADD_A_FORMAT.md`, all 5060 Ti against llama.cpp +b10660 Vulkan at `-ngl 99` on 1B-class vehicles, say: + +| format | pp512 ours / theirs | ratio | cite | +|---|---|---|---| +| IQ4_XS | 15334 / 17060 | 0.90x | `HOW_TO_ADD_A_FORMAT.md:1132` | +| Q2_K | 14544.0 / 16752.7 | 0.87x | `:774` | +| Q3_K | 14031 / 17509 | 0.80x | `:1088` | +| IQ4_NL | 15027.4 / 19177.8 | 0.78x | `:831` | +| IQ2_XXS | — | 0.78x | `:554` | +| IQ2_XS | 14023.3 / 18320.0 | 0.77x | `:606` | +| IQ3_S | 12539.6 / 17865 | 0.70x | `:1020` | +| IQ2_S | 12099.3 / 17377.5 | 0.70x | `:702` | +| IQ3_XXS | 12225.7 / 17807.7 | 0.69x | `:918` | +| Q4_K (the control) | — | 0.67x | `:987` | + +That is a 1.34x spread, not a band. Entry 34 was written on 2026-08-30 +(`followup_vulkan.md:543`) when only k4 and iq3s had Vulkan rows; six formats landed after it +and none updated it. The document contradicts itself at `HOW_TO_ADD_A_FORMAT.md:606-607` +("0.77x - above the 0.70x tier class") and `:753-754` ("0.87x - above the 0.69-0.78 tier +class"). + +**O15 — The "shared tier" diagnosis is refuted by our own Q8 rows.** +Q8_0 on the same tile, the same chain, the same box measures 3B pp 7737.2 = 100.6% of the +reference exe and tinyllama pp 20986 = about 103.5% (`followup_vulkan.md:211-213`). If the deficit +were the shared batch-GEMM tier, Q8 would show it. And the kernel-rate probes already point at +the decode: Q8 cm2 clocks 62.2/64.7 TF/s on gate/down (`followup_vulkan.md:176-178`) while Q4_K +clocks 35.8-38.2 TF/s, "about 70% of Q8-cm2's rate — the nibble+scale extraction" +(`followup_vulkan.md:223-224`); Q6_K sat at 9.3-13.4 TF/s until one byte4 dynamic select was +respelled, after which it hit 32.9 TF/s (`followup_vulkan.md:229-234`). + +The honest caveat is that these Q8 rows are 3B-shape and the per-format rows are 1B-shape, so +shape and decode are confounded in the evidence that exists. Nobody has run them apart. + +--- + +## 3. The deltas, ranked by estimated pp512 payoff on a 1B/3B shape + +Shape assumed throughout: dim 2048-3072, hidden 8192, 512 tokens, so the large tile +(BM 128, BN 256, BK 64) and split-k off on the wide GEMMs, split-k 2 on the narrow k/v +projections. + +### Delta 1 — Hoist the per-(row, sub-block) scale into workgroup memory, llama.cpp's `shAscales` form. Estimated payoff: 10-20% on Q4_K/Q5_K, less on the rest. + +Ours re-derives the scale for every decoded element: a 32-bit load from a second SSBO plus a +runtime-variable shift plus a mask plus a float convert, three times over for Q4_K +(`dasllama_vulkan_classes.das:5034-5038`). llama.cpp does that arithmetic once per (row, +sub-block) — 8 times per 256 elements per row — into `shAscales`, and the element decode is then +a single `vec2` read with no shifting (`dequant_funcs_cm2.glsl:484-523, 537-540`). That is a +32-fold reduction in scale-extraction ALU on the format that is currently our worst. + +This is the strongest hypothesis because Q4_K is simultaneously (a) our worst ratio at 0.67x +and (b) the only format llama.cpp gives a special optimisation to on the cm2 path. The +coincidence is unlikely. + +Cost to try: 8 sub-blocks times (BM + 2) times a `vec2` = 8320 bytes of workgroup memory +(`dequant_funcs_cm2.glsl:442-444`), which llama.cpp already spends without occupancy trouble on +the same tile. The fetch/store pipelining and the two barriers per 256 k +(`mul_mm_cm2.comp:363-374`, `dequant_funcs_cm2.glsl:486, 522`) come with it. + +**Measurement that proves or kills it.** `harness/vk_gemm_probe.das` already has the instrument: +the `k6x` arm varies one decode callback against spellings of itself with the tile held fixed, +and its `flat` variant is "current compose w/o scale reads" +(`D:\Work\daScript-iquant\modules\dasLLAMA\harness\vk_gemm_probe.das:466-469`). Run `k6x` at the +q6k gate and down shapes it already pins (9728x2560x512 and 2560x9728x512, +`vk_gemm_probe.das:1287-1288`) and read the `nil` to `flat` to shipped ladder. The `flat`-to- +shipped gap is the entire scale-plane cost in TF/s. If it is under 5%, delta 1 is dead. Then add +a k4 twin of that bisect — the file has no k4 scale-plane arm today, only a k4 rate arm +(`vk_gemm_probe.das:357`). +Caveat: arm `DAS_TUNE_MANIFEST` or the checked-in +`performance/last_known_good_sidecar.json` first, or a bare probe run re-execs into a full +retune (`ARCHITECTURE_MEASUREMENT.md:53-56`). + +### Delta 2 — Fold the scale plane back into the quant block, so the decode touches one buffer. Estimated payoff: 5-15%, uniform across formats. + +llama.cpp's callback gets a `buffer_reference` the driver already resolved and reads d, dmin and +all 12 scale bytes as one 16-byte aligned `uvec4` (`dequant_funcs_cm2.glsl:429-431, 461`). Ours +carries two independent address streams and a 20-byte scale stride that is not 16-byte aligned +(`dasllama_vulkan_classes.das:4804-4805, 5034`). Two streams double the distinct addresses in +flight per tile and halve the chance the block-load path coalesces them. + +This one is uniform across formats, which is what would explain the part of the deficit that +IQ4_XS still shows at its otherwise-matched decode-load count. + +**Measurement.** The probe's `cm2x` arm already carries a `push` variant that moves the scale +base off the workgroup read onto a push constant +(`vk_gemm_probe.das:215-216`), and a `flat` variant with no scale read at all +(`:72-74`). Those bracket the address-arithmetic half. The buffer-count half needs a new arm: a +k4 fixture whose `ws` contents are interleaved into the `wq` plane and read through the same +block pointer. Compare TF/s on one shape (d 2048, K 2048, cnt 512, large tile, split off via +`DASLLAMA_CM2_SPLITK=1`). Kill it if the interleaved arm is within 3%. + +### Delta 3 — Split the end-to-end budget before spending anything on the kernel. Estimated payoff: none directly; it decides whether deltas 1, 2 and 4-8 are worth doing at all. + +Every ratio in the table above is end-to-end pp512 tokens per second, not GEMM time. An +end-to-end 0.67x does not prove the GEMM is 0.67x. The chain around it carries 452 nodes and 367 +barriers per window on the 3B (`followup_vulkan.md:246-248`), plus the f16 conversion dispatches +(`dasllama_vulkan_prefill.das:760-771, 852-863, 883-892, 905-914`) and the hazard fences +(`dasllama_vulkan_common.das:2587-2620`). + +**Measurement.** Two halves, both cheap. +On llama.cpp: `set GGML_VK_PERF_LOGGER=1` and run the same model and prompt through +`llama-bench -p 512 -n 0 -ngl 99`. It prints one line per op keyed by shape — +`MUL_MAT q4_K m=2048 n=512 k=2048: N x T us = T us (G GFLOPS/s)` +(`ggml-vulkan.cpp:2278-2297, 2240-2254`). +On ours: the prefill already timestamps every role (`pfq_ts`, `dasllama_vulkan_prefill.das:391`, +roles reported at `:983-985`). Sum the GEMM roles and compare against the sum of llama.cpp's +`MUL_MAT` rows for the same forward. +If our MUL_MAT total is at parity and the wall clock is not, the deficit is outside the GEMM and +deltas 1-2 are the wrong target. If our MUL_MAT total is 0.67x, the deficit is in the kernel and +delta 1 leads. + +### Delta 4 — Pad the activation plane's N to the tile width and keep the fast path. Estimated payoff: zero at exactly 512 tokens, up to about `1/ceil(N/BN)` of the GEMM at other token counts. + +llama.cpp pads B's N to the tile width in the prealloc buffer specifically so the last token +column still takes the unclamped path (`ggml-vulkan.cpp:9319`, gate at +`mul_mm_cm2.comp:345`). Ours sizes the plane to `cnt` and drops the partial column onto the fully +clamped edge path (`dasllama_vulkan_classes.das:4893, 4901`). At 512 tokens with BN 256 there is +no partial column, so this is invisible in the pp512 board — and visible in every real prompt +whose length is not a multiple of 256. + +**Measurement.** Run pp at 512 and at 500 tokens, ours and llama.cpp, same model. Our ratio +should degrade at 500 and theirs should not. If both degrade equally, the edge path is not the +cause. + +### Delta 5 — Narrow the tail accumulator (`enable_smaller_matrices`). Estimated payoff: same trigger as delta 4, roughly half its size. + +llama.cpp allocates a `BN/2` or `BN/4` accumulator for the tail column +(`mul_mm_cm2.comp:36-38, 367, 410`), so a 64-token remainder does not drag a 256-wide +accumulator through the whole k loop. We have no analogue. Fix delta 4 first; this is the +refinement on top of it. + +**Measurement.** Only meaningful after delta 4 lands. Same 500-token comparison, with and without +a narrowed tail stamp. + +### Delta 6 — Reduce the inner unroll from 8 to 4. Estimated payoff: 0 to 5%, sign unknown. + +Ours unrolls 8 deep, 512 k per outer iteration (`dasllama_vulkan_classes.das:4909-4916`); +llama.cpp unrolls 4 at `QUANT_K = 256` (`mul_mm_cm2.comp:341-342`). With an f16 128x256 +accumulator already costing 64 registers per invocation, an 8-deep window the compiler chooses +to software-pipeline could cost an occupancy tier. The architecture doc records that a related +codegen spelling change was worth 27% of prefill throughput on this exact box +(`ARCHITECTURE_GPU_VULKAN.md:126-130`), so codegen sensitivity here is established, not +speculative. + +**Measurement.** Build the k4 large class at unroll 4 and at 2 alongside the shipped 8 and time +all three on one shape through the probe's shape arms. Read the driver's register/occupancy +report if Nsight is attached; otherwise the timing alone decides. + +### Delta 7 — No four-wide decode callback. Estimated payoff: zero today, large the day glslc is upgraded. + +llama.cpp ships a `_v` twin of every decode function returning `f16vec4` +(`dequant_funcs_cm2.glsl:576-623` for Q4_K, `:778-805` for Q6_K, `:1002-1025` for IQ2_XS, and so +on for every format), wired through `GGML_VULKAN_COOPMAT2_DECODE_VECTOR` +(`mul_mm_cm2.comp:82-86`, `vulkan-shaders-gen.cpp:480-484`). The local glslc cannot build it +(`tbo_build_vk.log`), so llama.cpp runs scalar here — but the moment that toolchain moves, +llama.cpp's callback count drops fourfold and every delta above gets 4x harder to close. Our +`[spirv_decode]` surface has no vec4 form. + +**Measurement.** None needed to establish the risk. To size it, build llama.cpp with a glslc that +supports the extension and re-run the same per-format board; the delta between the two llama.cpp +builds is what we would owe. + +### Delta 8 — The per-workgroup prologue and the computed codebooks. Estimated payoff: under 2% on the dense 1B shape; real on MoE grids. + +Five `sched` loads, a workgroup broadcast and a barrier before the first tile load +(`dasllama_vulkan_classes.das:3972-3976, 4849-4851, 4887`), plus synthesising 256 to 2048 +codebook words with arithmetic rather than copying them (`:4861-4886`). On a 32-workgroup dense +dispatch this amortises away. On a MoE expert grid with many small dispatches it does not, which +is the GEMV twin of the problem `followup_vulkan.md:548-557` (entry 35) already flags for tg. + +**Measurement.** A variant that hardcodes the single-region dense case — region 0, workgroup base +0, no `sched` read — A/B against the shipped class on the same shape. If the dense arm gains +under 2%, leave it alone and route the effort to the MoE grids instead. + +**One correction to file regardless of the outcome.** `followup_vulkan.md:540-546` (entry 34) +is wrong on three counts against the repo's own tables: the band is 0.67-0.90x not +0.67-0.70x; "the gap is the shared batch-GEMM tier, not any one format's decode" is contradicted +by Q8 at 100.6%/103.5% (`:211-213`) and by the 62 versus 36-38 TF/s tile rates (`:176-178`, +`:223-224`); and "tg holds parity or better" is contradicted by IQ2_XS tg at 0.54x +(`HOW_TO_ADD_A_FORMAT.md:607`), which entry 35 itself documents. + +--- + +## 4. What `test-backend-ops perf -o MUL_MAT` gives, and the exact flags + +**The binary is already built:** +`D:\Work\llama.cpp\build-vulkan\bin\Release\test-backend-ops.exe` +(`C:\Users\Boris\.claude\jobs\7f9893d9\tmp\tbo_build_vk.log`, final line). + +**Command:** + +``` +D:\Work\llama.cpp\build-vulkan\bin\Release\test-backend-ops.exe perf -o MUL_MAT -b Vulkan0 +``` + +- `perf` selects performance mode (`tests/test-backend-ops.cpp:11013-11014`). +- `-o MUL_MAT` filters by `ggml_op_desc` (`:11021-11027`, usage at `:10992-10993`). +- `-b Vulkan0` filters by `ggml_backend_dev_name`; the Vulkan device name is + `GGML_VK_NAME + index` = `"Vulkan0"` (`:11098`, `ggml-vulkan.cpp:7001`, + `ggml/include/ggml-vulkan.h:10`). +- `--output csv` or `--output sql` for machine-readable rows (`:11040-11047`). +- `-p ` filters on the case's `vars()` string, which for MUL_MAT is + `type_a,type_b,m,n,k,bs,nr,per,k_v,o,src_overlap` (`:4490-4492`). Example: + `-p "type_a=q4_K,.*n=512,k=14336"`. + +**What each row reports:** average microseconds per op, GFLOPS computed as +`2*m*n*k*bs[0]*nr[0]*bs[1]*nr[1]` over wall time, and the run count +(`:4510-4513`, `:1639-1646`). The harness sizes the run count to about 100 GFLOP of work on a +GPU backend, duplicates the op that many times into one graph, does a warmup compute, then loops +whole-graph submits until at least one second has elapsed (`:1569-1601`, `:1620-1636`). + +**What it gives us as a reference row:** a per-kernel GFLOPS number for llama.cpp's cm2 GEMM at a +fixed shape, isolated from the rest of a forward pass, on the same device and driver — the exact +counterpart to `vk_gemm_probe.das`'s per-shape TF/s rows. Because the graph repeats one node, +the f32-to-f16 activation conversion is converted once and reused across the duplicates +(`ggml-vulkan.cpp:9441-9453`), so the reported GFLOPS is close to pure GEMM. + +**The limitation that matters, and the workaround.** Perf mode does not use the correctness test +set; it uses a fixed list built by `make_test_cases_perf` (`:10151`, dispatched at `:10727-10728`). +The only quantized MUL_MAT rows at 512 tokens in that list are + +``` +for (int bs : {1, 2, 3, 4, 5, 8, 512}) + for (ggml_type type_a : all_types) + test_mul_mat(type_a, GGML_TYPE_F32, 4096, bs, 14336, {1,1}, {1,1}); +``` +(`:10321-10323`) + +so the available shape is **m 4096, n 512, k 14336** — a Llama-8B FFN shape, not a 1B/3B one. +`-p` can only filter what is already in the list; it cannot create a 2048x512x2048 or +3072x512x8192 row. Getting our shape needs either a one-line addition to that loop (a local +patch to a tree this task treats as read-only) or `--test-file` with a graph exported by +`test-export-graph-ops` (`:11058-11061`, usage at `:10997`). + +Two practical notes. First, the 4096x512x14336 row is still a useful reference: it takes the +same aligned large pipeline (`tiles_m = 32*4 = 128 > 36`, so `prefer_large`) and split-k 1 +(128 tiles, well over `36*2/3`), which is the configuration a 1B prefill also lands on. Second, +for a shape-matched comparison the cheaper route is `GGML_VK_PERF_LOGGER=1` on a real +`llama-bench -p 512 -n 0 -ngl 99` run: it emits per-shape `MUL_MAT m=.. n=.. k=..` rows +with GFLOPS from the actual model geometry (`ggml-vulkan.cpp:2278-2297`), which is what delta 3 +needs anyway. + +--- + +## Summary table + +| dimension | llama.cpp cm2 | ours | +|---|---|---| +| tile large / medium / small | 128x256x64 / 128x128x64 / 32x64x128 | 128x256x64 / 128x128x64 / 128x32x64 | +| workgroup | 256 invocations | 256 invocations | +| accumulate | f16 (f32 on PREC_F32) | f16, clamped, widened to f32 at store | +| A operand | quantized, decoded in the driver's block load | same | +| B operand | f16, one cached conversion dispatch | f16, one conversion per sibling group, SwiGLU fused | +| scales | inside the block; Q4_K/Q5_K hoisted to shared memory | separate SSBO, re-extracted per element with variable shifts | +| barriers per k-step | 0 (2 per 256 k on Q4_K/Q5_K only) | 0 | +| loads per k-step | 1 A + 1 B | 1 A + 1 B | +| unroll | 4 x BK | 8 x BK | +| split-k | k>=2048, cores/tiles, cap 8, 256-aligned | identical rule | +| N padding | buffer padded to tile width | none (large/medium); small tile rounds to 32 | +| tail accumulator | BN/2, BN/4 | none | +| grid | 3-D from `gl_WorkGroupID` | 1-D plus an SSBO region record | +| batching | `gl_WorkGroupID.z` plus a separate MUL_MAT_ID shader | region list in the same kernel | +| vec4 decode | present, gated on glslc (absent locally) | absent | From a29ce1c56cd045a60e64452f2662398e945d2050 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 00:36:40 -0700 Subject: [PATCH 002/123] kernel parity, CPU item 2: sign="vec" for iq3s and iq2s - the sign column negates whole vectors The panel holds raw grid words; per block one 32-byte load of the plane's sign column (the 8-weight sign bytes sit at row stride 4 for these two formats), a constant shuffle + and + cmpeq builds the byte mask, xor + sub negates the vector. One thread, m=4096 k=14336: iq3s decode 11581 -> 7732 us (the reference exe 10340), iq2s 12413 -> 7076 us (5074). Every variant bit-exact in gen_tune_probe TEST mode. The five gather emitters collapse into one gather over per-format decode functions. gather="reg" (compose the vector with insertelement, no panel) was measured first and dropped: 90.6 ns/superblock against the panel's 49.1, from 1025 insertelement chains and a 5x code body. kq_kernel_bench's tile arm hands the five grid formats the caller-unpacked panel form they read. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../benchmarks/matmul/kq_kernel_bench.das | 15 +- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 464 +++++++----------- .../dasLLAMA/dasllama/dasllama_math_gen.das | 8 + modules/dasLLAMA/followup_general.md | 3 + plans/kernel_parity_pass.md | 23 +- plans/kernel_parity_research_vk.md | 108 ++-- 6 files changed, 268 insertions(+), 353 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 8be819d795..d7b74b229b 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -126,8 +126,12 @@ def layout_mrs(f : KqFmt) : table { return <- mrs } +//! the five grid formats' TILE reads a caller-unpacked panel (256 bytes per row per superblock), not the packed plane the gemv gathers from +def tile_kq_bytes_per_sb(f : KqFmt) : int64 => (f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq2s || f == KqFmt.iq2xs || f == KqFmt.iq2xxs) ? 256l : kq_qsb(f) + struct Planes { kq : array + tkq : array // the tile arm's weight plane (the unpacked panel form where the format has one) ks : array xq : array xs : array @@ -143,6 +147,7 @@ def sized(var a : array; n : int64) { def fill_planes(var p : Planes; f : KqFmt; n, d, ntok : int64) { let nsb = n / 256l p.kq |> sized(d * nsb * kq_qsb(f)) + p.tkq |> sized(d * nsb * tile_kq_bytes_per_sb(f)) p.ks |> sized(d * nsb * kq_ssb(f)) p.xq |> sized(ntok * n) p.xs |> sized(ntok * n / 32l) @@ -153,6 +158,10 @@ def fill_planes(var p : Planes; f : KqFmt; n, d, ntok : int64) { seed = seed * 1664525u + 1013904223u b = uint8(seed >> 24u) } + for (b in p.tkq) { + seed = seed * 1664525u + 1013904223u + b = uint8(seed >> 24u) + } for (b in p.ks) { seed = seed * 1664525u + 1013904223u b = uint8(seed >> 24u) @@ -212,13 +221,13 @@ def bench_tile(var rows : int&; cfg : BenchArgs; f : KqFmt; var p : Planes) { let d = int64(cfg.d) let ntok = int64(cfg.ntok) let nsb = n / 256l - let qsb = kq_qsb(f) + let qsb = tile_kq_bytes_per_sb(f) let ssb = kq_ssb(f) - let plane_bytes = long_length(p.kq) + long_length(p.ks) + let plane_bytes = long_length(p.tkq) + long_length(p.ks) var mrs <- layout_mrs(f) var vs <- tile_variants(f) unsafe { - let kqp = addr(p.kq[0]) + let kqp = addr(p.tkq[0]) let ksp = addr(p.ks[0]) let xqp = addr(p.xq[0]) let xsp = addr(p.xs[0]) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 2d13a33379..d5a665a2bb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -90,6 +90,7 @@ struct private TilePerm { mr : int = GEMM_REFERENCE_MR() gkstep : int = 1 dotPrim : string = "sdot" + sign : string = "scalar" // iq3s/iq2s gemv: "scalar" = signs applied per grid dword in the gather, "vec" = raw words in the panel, one masked negate per weight vector off the plane's sign column width : int = 128 bias : int = 0 // 128 = bias128: grp bytes baked as w^0x80, plain vpdpbusd dots pipe : int = 0 // 1 = software-pipelined amx fold (slice K I4): double-buffered C @@ -110,6 +111,8 @@ def private parse_perm(gc : LlvmCodeCtx) : TilePerm { if (gv is tInt) { p.gkstep = gv as tInt } let dv = find_arg(gc.ann.arguments, "dot") if (dv is tString) { p.dotPrim = dv as tString } + let sgv = find_arg(gc.ann.arguments, "sign") + if (sgv is tString) { p.sign = sgv as tString } let wv = find_arg(gc.ann.arguments, "width") if (wv is tInt) { p.width = wv as tInt } let bv = find_arg(gc.ann.arguments, "bias") @@ -125,6 +128,7 @@ def private parse_perm(gc : LlvmCodeCtx) : TilePerm { // fall back together on every rail, or repack and kernel desync (the two-function-stamp // contract). Rails: knob validity, schema, per-ISA leg, geometry, vreg budget, dot availability. def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint:STYLE037,STYLE038 — the flat decline ladder IS the perm contract; splitting hides which leg refused + if (p.sign != "scalar" && p.sign != "vec") return true if (p.dotPrim == "amx_int8") { // the TMUL leg: 2×2 TDPBSSD macro over the grp16/kg4 plane; kstep pinned 1 (Q8's per-block scale boundary); Linux-only (arch_prctl enable) if ((p.kstep != 1) @@ -246,6 +250,7 @@ struct private TileEmit { abs_ty : LLVMOpaqueType? psign_decl : LLVMOpaqueValue? // x64 width-256 sign apply (VPSIGNB) iq3s_panel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) + signVec : bool // sign="vec": the gemv panel holds raw grid words; the signs land as one masked negate per weight vector iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) iq3s_smask_g : LLVMOpaqueValue? // iq3s gemv: the [16 x i32] sign-nibble byte-mask global (iq3xxs shares it) iq3xxs_ksigns_g : LLVMOpaqueValue? // iq3xxs gemv: the [128 x i32] ksigns_iq2xs global (7-bit index -> sign byte) @@ -1294,61 +1299,173 @@ def private iq3s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { te.iq3s_smask_g = smask } -// iq3s gemv: gather superblock `sbi`'s grid words + signs off the PACKED grp planes into the -// alloca panel — one emitted loop over the mr rows (32 grid words unrolled per iteration); a -// single straight-line loop block, so the phi's back edge is the block itself. -def private emit_iq3s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { +// The five panel formats' per-dword decode - one grid word of the row whose packed grp region +// starts at byte `base`, for block `blk`, sign group `l`, half `h`; loads shared across (blk, l, h) fold in LLVM's CSE. +def private wg_byte(var te : TileEmit; var base : LLVMOpaqueValue?; off : int) : LLVMOpaqueValue? { + var p = LLVMBuildGEP2(te.builder, te.types.t_int8, te.wg, LLVMBuildAdd(te.builder, base, te.types->ConstI64(uint64(off)), ""), "") + return LLVMBuildZExt(te.builder, LLVMBuildLoad2Aligned(te.builder, te.types.t_int8, p, 1u, ""), te.types.t_int32, "") +} + +def private wg_i32(var te : TileEmit; var base : LLVMOpaqueValue?; off : int) : LLVMOpaqueValue? { + var p = LLVMBuildGEP2(te.builder, te.types.t_int8, te.wg, LLVMBuildAdd(te.builder, base, te.types->ConstI64(uint64(off)), ""), "") + return LLVMBuildLoad2Aligned(te.builder, te.types.t_int32, p, 4u, "") +} + +//! a packed-plane byte position to its grp column offset (4-byte columns, mr rows each) +def private wg_col(mr, pos : int) : int => (pos / 4) * mr * 4 + pos % 4 + +def private ksigns_word(var te : TileEmit; var sidx : LLVMOpaqueValue?) : LLVMOpaqueValue? { + var p = LLVMBuildGEP2(te.builder, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") + return LLVMBuildLoad2Aligned(te.builder, te.types.t_int32, p, 4u, "") +} + +//! grid[gidx] with sign nibble `h` of `sgn` applied: (g ^ m) + (m & 0x01010101) negates the masked +//! bytes. Under sign="vec" the word stays raw - the block emitter negates whole vectors instead. +def private signed_grid_word(var te : TileEmit; var gidx, sgn : LLVMOpaqueValue?; h : int) : LLVMOpaqueValue? { + let b = te.builder + var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gidx, "") + var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "") + if (te.signVec) { + return gw + } + let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) + var mv = h == 0 ? LLVMBuildAnd(b, sgn, c15, "") : LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") + var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv, "") + var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") + return LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, LLVMConstInt(te.types.t_int32, 0x01010101ul, 0), ""), "") +} + +//! sign="vec" needs the 8-weight sign bytes plane-resident at row stride 4: iq3s and iq2s only +def private sign_vec_declines(p : TilePerm; fmt : int) : bool => p.sign == "vec" && fmt != 33 && fmt != 23 + +//! the grp column (4 bytes per row) holding block `blk`'s four sign bytes +def private kq_sign_col(kq, blk : int) : int => (kq == 33 ? 18 : 8) + blk + +//! one masked negate per weight vector: byte (r, k) of the mask is 0xFF when bit (h*4 + k) of row r's +//! sign byte for group `l` is set - a constant shuffle broadcasts each row's byte to its four lanes +def private apply_sign_col(var te : TileEmit; var w, col : LLVMOpaqueValue?; l, h : int; name : string) : LLVMOpaqueValue? { + let b = te.builder + let w8 = te.width / 8 + let idx <- [for (i in range(w8)); (i / 4) * 4 + l] + var bytes = LLVMBuildShuffleVector(b, te.types, col, col, idx, "sgb") + var bits : array + bits |> reserve(w8) + for (i in range(w8)) { + bits |> push(LLVMConstInt(te.types.t_int8, uint64(1 << (h * 4 + i % 4)), 0)) + } + var bitsv = LLVMConstVector(array_data_ptr(bits), uint(w8)) + var m = LLVMBuildSExt(b, LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntEQ, LLVMBuildAnd(b, bytes, bitsv, ""), bitsv, ""), te.vwi8, "sgm") + return LLVMBuildSub(b, LLVMBuildXor(b, w, m, ""), m, name) +} + +//! iq3s: 8-bit qs byte + the block's qh bit = the 9-bit grid index; explicit sign byte per 8 weights +def private decode_iq3s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { let b = te.builder let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 104)), "iq3s.wb") + var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) + var sgn = wg_byte(te, base, wg_col(mr, 72 + blk * 4 + l)) + var qv = wg_byte(te, base, wg_col(mr, blk * 8 + 2 * l + h)) + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - h - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 256ul, 0), "") + return signed_grid_word(te, LLVMBuildOr(b, qv, hb, ""), sgn, h) +} + +//! iq3xxs: 8-bit index into the halved grid; the block's aux32 carries four 7-bit ksigns codes +def private decode_iq3xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + let b = te.builder + let mr = te.interleave + var aux = wg_i32(te, base, (16 + blk) * mr * 4) + var sidx = LLVMBuildAnd(b, l == 0 ? aux : LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), LLVMConstInt(te.types.t_int32, 127ul, 0), "") + var iv = wg_byte(te, base, wg_col(mr, blk * 8 + 2 * l + h)) + return signed_grid_word(te, iv, ksigns_word(te, sidx), h) +} + +//! iq2s: qs byte | the qh 0x300 window = the 10-bit index, doubled into the u64 grid's word pair; explicit sign byte +def private decode_iq2s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + let b = te.builder + let mr = te.interleave + var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) + var sgn = wg_byte(te, base, wg_col(mr, 32 + blk * 4 + l)) + var iv0 = wg_byte(te, base, wg_col(mr, blk * 4 + l)) + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 0x300ul, 0), "") + var idx2 = LLVMBuildShl(b, LLVMBuildOr(b, iv0, hb, ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + return signed_grid_word(te, gi, sgn, h) +} + +//! iq2xs: the u16 word's low 9 bits index the u64 grid's word pair, its high 7 bits the ksigns code +def private decode_iq2xs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + let b = te.builder + let mr = te.interleave + let lp = (blk * 4 + l) * 2 + var qlo = wg_byte(te, base, wg_col(mr, lp)) + var qhi = wg_byte(te, base, wg_col(mr, lp) + 1) + var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "") + var idx2 = LLVMBuildShl(b, LLVMBuildAnd(b, w16, LLVMConstInt(te.types.t_int32, 511ul, 0), ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var sgn = ksigns_word(te, LLVMBuildLShr(b, w16, LLVMConstInt(te.types.t_int32, 9ul, 0), "")) + var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + return signed_grid_word(te, gi, sgn, h) +} + +//! iq2xxs: block b's grid bytes (column 2b) index the u64 grid's word pair; its aux32 (column 2b+1) carries the ksigns codes +def private decode_iq2xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + let b = te.builder + let mr = te.interleave + var aux = wg_byte(te, base, (2 * blk + 1) * mr * 4) + for (k in range(1, 4)) { + aux = LLVMBuildOr(b, aux, LLVMBuildShl(b, wg_byte(te, base, (2 * blk + 1) * mr * 4 + k), LLVMConstInt(te.types.t_int32, uint64(8 * k), 0), ""), "") + } + var idx2 = LLVMBuildShl(b, wg_byte(te, base, 2 * blk * mr * 4 + l), LLVMConstInt(te.types.t_int32, 1ul, 0), "") + var sidx = LLVMBuildAnd(b, LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), LLVMConstInt(te.types.t_int32, 127ul, 0), "") + var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + return signed_grid_word(te, gi, ksigns_word(te, sidx), h) +} + +def private decode_kq_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + if (te.kq == 33) return decode_iq3s_dword(te, base, blk, l, h) + if (te.kq == 34) return decode_iq3xxs_dword(te, base, blk, l, h) + if (te.kq == 23) return decode_iq2s_dword(te, base, blk, l, h) + if (te.kq == 24) return decode_iq2xs_dword(te, base, blk, l, h) + return decode_iq2xxs_dword(te, base, blk, l, h) +} + +//! bytes per row per superblock of a panel format's packed grp region +def private kq_gather_stride(kq : int) : int => kq == 33 ? 104 : (kq == 34 ? 96 : (kq == 23 ? 72 : 64)) + +//! byte offset of the (blk, l, h) dword group in the panel: two 32-group halves, mr dwords per group +def private panel_group_off(mr, blk, l, h : int) : int { + let e = l * 8 + h * 4 + return (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 +} + +// gemv, gather="panel": superblock `sbi` off the PACKED grp planes into the alloca panel - one +// emitted loop over the mr rows (64 decoded dwords per iteration), a single straight-line loop +// block, so the phi's back edge is the block itself. +def private emit_kq_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { + let b = te.builder + let mr = te.interleave + var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr) * uint64(kq_gather_stride(te.kq))), "kq.wb") var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3s.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3s.cont") + var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "kq.gath") + var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "kq.cont") LLVMBuildBr(b, head) LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq3s.r") + var rPhi = LLVMBuildPhi(b, te.types.t_int64, "kq.r") var preVals <- [te.types->ConstI64(0ul)] var preBlocks <- [pre] LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq3s.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq3s.base") - let c256 = LLVMConstInt(te.types.t_int32, 256ul, 0) - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) + var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "kq.r4") + var base = LLVMBuildAdd(b, wbP, rT4, "kq.base") for (blk in range(8)) { - let hp = 64 + blk - var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((hp / 4) * mr * 4 + hp % 4)), ""), "") - var qh = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "iq3s.qh{blk}") for (l in range(4)) { - let sp = 72 + blk * 4 + l - var sgp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((sp / 4) * mr * 4 + sp % 4)), ""), "") - var sgn = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, sgp, 1u, ""), te.types.t_int32, "iq3s.sg{blk}_{l}") - var iv : LLVMOpaqueValue? [2] - for (h in range(2)) { - let qp1 = blk * 8 + 2 * l + h - var qpp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((qp1 / 4) * mr * 4 + qp1 % 4)), ""), "") - var qv = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qpp, 1u, ""), te.types.t_int32, "") - var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - h - 2 * l), 0), ""), c256, "") - iv[h] = LLVMBuildOr(b, qv, hb, "iq3s.i{blk}_{l}_{h}") - } - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") for (h in range(2)) { - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, iv[h], "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq3s.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq3s.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") + var w = decode_kq_dword(te, base, blk, l, h) + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(panel_group_off(mr, blk, l, h))), ""), "") LLVMBuildStore(b, w, dp) } } } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq3s.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq3s.more") + var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "kq.rn") + var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "kq.more") LLVMBuildCondBr(b, more, head, cont) var loopVals <- [rNext] var loopBlocks <- [head] @@ -1412,64 +1529,6 @@ def private iq3xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { te.iq3s_smask_g = smask } -// iq3xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the -// iq3s gather's shape with the aux32 sign word (ONE aligned i32 column load per block) and -// ksigns lookups in place of the explicit sign bytes, and no 9th grid-index bit. -def private emit_iq3xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { - let b = te.builder - let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 96)), "iq3x.wb") - var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3x.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq3x.cont") - LLVMBuildBr(b, head) - LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq3x.r") - var preVals <- [te.types->ConstI64(0ul)] - var preBlocks <- [pre] - LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq3x.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq3x.base") - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let c127 = LLVMConstInt(te.types.t_int32, 127ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) - for (blk in range(8)) { - var axp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((16 + blk) * mr * 4)), ""), "") - var aux = LLVMBuildLoad2Aligned(b, te.types.t_int32, axp, 4u, "iq3x.aux{blk}") - for (l in range(4)) { - var sidx = LLVMBuildAnd(b, l == 0 ? aux : LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), c127, "") - var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") - var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq3x.sg{blk}_{l}") - var iv : LLVMOpaqueValue? [2] - for (h in range(2)) { - let qp1 = blk * 8 + 2 * l + h - var qpp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((qp1 / 4) * mr * 4 + qp1 % 4)), ""), "") - iv[h] = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qpp, 1u, ""), te.types.t_int32, "iq3x.i{blk}_{l}_{h}") - } - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") - for (h in range(2)) { - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, iv[h], "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq3x.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq3x.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") - LLVMBuildStore(b, w, dp) - } - } - } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq3x.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq3x.more") - LLVMBuildCondBr(b, more, head, cont) - var loopVals <- [rNext] - var loopBlocks <- [head] - LLVMAddIncoming(rPhi, loopVals, loopBlocks) - LLVMPositionBuilderAtEnd(b, cont) -} // The iq2s gemv gather globals: the u64 grid as a [2048 x i32] private constant (1024 entries // as low/high word pairs), plus the shared smask table (magnitudes {8, 25, 43} are nonzero - @@ -1512,66 +1571,6 @@ def private iq2s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { te.iq3s_smask_g = smask } -// iq2s gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the -// iq3s gather's shape with the 10-bit index (qs byte | qh's 0x300 window) doubled into the -// u64 grid's low/high word pair; the sign byte is explicit like iq3s's. -def private emit_iq2s_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { - let b = te.builder - let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 72)), "iq2s.wb") - var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2s.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2s.cont") - LLVMBuildBr(b, head) - LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2s.r") - var preVals <- [te.types->ConstI64(0ul)] - var preBlocks <- [pre] - LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2s.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq2s.base") - let c768 = LLVMConstInt(te.types.t_int32, 0x300ul, 0) - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) - for (blk in range(8)) { - let hp = 64 + blk - var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((hp / 4) * mr * 4 + hp % 4)), ""), "") - var qh = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "iq2s.qh{blk}") - for (l in range(4)) { - let sp = 32 + blk * 4 + l - var sgp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((sp / 4) * mr * 4 + sp % 4)), ""), "") - var sgn = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, sgp, 1u, ""), te.types.t_int32, "iq2s.sg{blk}_{l}") - let ip = blk * 4 + l - var ipp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((ip / 4) * mr * 4 + ip % 4)), ""), "") - var iv0 = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, ipp, 1u, ""), te.types.t_int32, "") - var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), c768, "") - var idx = LLVMBuildOr(b, iv0, hb, "iq2s.i{blk}_{l}") - var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") - for (h in range(2)) { - var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2s.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2s.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") - LLVMBuildStore(b, w, dp) - } - } - } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2s.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2s.more") - LLVMBuildCondBr(b, more, head, cont) - var loopVals <- [rNext] - var loopBlocks <- [head] - LLVMAddIncoming(rPhi, loopVals, loopBlocks) - LLVMPositionBuilderAtEnd(b, cont) -} // The iq2xs gemv gather globals: the u64 grid as a [1024 x i32] private constant (512 entries // as low/high word pairs), the shared ksigns table, and the shared smask. @@ -1683,127 +1682,7 @@ def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { te.iq3s_smask_g = smask } -// iq2xs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - the -// iq2s gather's two-word u64 form with the u16 qs word's 9-bit index and its high-7-bit -// KSIGNS lookup in place of explicit sign bytes (the iq3xxs sign path). -def private emit_iq2xs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { - let b = te.builder - let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 64)), "iq2x.wb") - var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2x.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2x.cont") - LLVMBuildBr(b, head) - LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2x.r") - var preVals <- [te.types->ConstI64(0ul)] - var preBlocks <- [pre] - LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2x.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq2x.base") - let c511 = LLVMConstInt(te.types.t_int32, 511ul, 0) - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) - for (blk in range(8)) { - for (l in range(4)) { - let lp = (blk * 4 + l) * 2 - var qlp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((lp / 4) * mr * 4 + lp % 4)), ""), "") - var qlo = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qlp, 1u, ""), te.types.t_int32, "") - var qhp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((lp / 4) * mr * 4 + lp % 4 + 1)), ""), "") - var qhi = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, qhp, 1u, ""), te.types.t_int32, "") - var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "iq2x.w16{blk}_{l}") - var idx = LLVMBuildAnd(b, w16, c511, "iq2x.i{blk}_{l}") - var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var sidx = LLVMBuildLShr(b, w16, LLVMConstInt(te.types.t_int32, 9ul, 0), "") - var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") - var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq2x.sg{blk}_{l}") - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") - for (h in range(2)) { - var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2x.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2x.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") - LLVMBuildStore(b, w, dp) - } - } - } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2x.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2x.more") - LLVMBuildCondBr(b, more, head, cont) - var loopVals <- [rNext] - var loopBlocks <- [head] - LLVMAddIncoming(rPhi, loopVals, loopBlocks) - LLVMPositionBuilderAtEnd(b, cont) -} -// iq2xxs gemv: gather superblock `sbi` off the PACKED grp planes into the alloca panel - block -// b's grid BYTES (column 2b) double into the u64 grid's word pair, its aux32 (column 2b+1) -// carries the 7-bit KSIGNS indices (the per-32 scale nibble rides the scale row, not the gather). -def private emit_iq2xxs_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { - let b = te.builder - let mr = te.interleave - var wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * 64)), "iq2xx.wb") - var pre = LLVMGetInsertBlock(b) - var head = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2xx.gath") - var cont = LLVMAppendBasicBlockInContext(te.gctx, te.gfn, "iq2xx.cont") - LLVMBuildBr(b, head) - LLVMPositionBuilderAtEnd(b, head) - var rPhi = LLVMBuildPhi(b, te.types.t_int64, "iq2xx.r") - var preVals <- [te.types->ConstI64(0ul)] - var preBlocks <- [pre] - LLVMAddIncoming(rPhi, preVals, preBlocks) - var rT4 = LLVMBuildMul(b, rPhi, te.types->ConstI64(4ul), "iq2xx.r4") - var base = LLVMBuildAdd(b, wbP, rT4, "iq2xx.base") - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - let c127 = LLVMConstInt(te.types.t_int32, 127ul, 0) - let cneg = LLVMConstInt(te.types.t_int32, 0x01010101ul, 0) - for (blk in range(8)) { - var a0p = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((2 * blk + 1) * mr * 4)), ""), "") - var aux = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, a0p, 1u, ""), te.types.t_int32, "") - for (k in range(1, 4)) { - var akp = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64((2 * blk + 1) * mr * 4 + k)), ""), "") - var ab = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, akp, 1u, ""), te.types.t_int32, "") - aux = LLVMBuildOr(b, aux, LLVMBuildShl(b, ab, LLVMConstInt(te.types.t_int32, uint64(8 * k), 0), ""), "iq2xx.aux{blk}_{k}") - } - for (l in range(4)) { - var ip = LLVMBuildGEP2(b, te.types.t_int8, te.wg, LLVMBuildAdd(b, base, te.types->ConstI64(uint64(2 * blk * mr * 4 + l)), ""), "") - var idx = LLVMBuildZExt(b, LLVMBuildLoad2Aligned(b, te.types.t_int8, ip, 1u, ""), te.types.t_int32, "iq2xx.i{blk}_{l}") - var idx2 = LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var sidx = LLVMBuildAnd(b, LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), c127, "") - var ksp2 = LLVMBuildGEP2(b, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") - var sgn = LLVMBuildLoad2Aligned(b, te.types.t_int32, ksp2, 4u, "iq2xx.sg{blk}_{l}") - var mv : LLVMOpaqueValue? [2] - mv[0] = LLVMBuildAnd(b, sgn, c15, "") - mv[1] = LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") - for (h in range(2)) { - var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gi, "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "iq2xx.g{blk}_{l}_{h}") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv[h], "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - var w = LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, cneg, ""), "iq2xx.w{blk}_{l}_{h}") - let e = l * 8 + h * 4 - let dd = (e < 16 ? 0 : 128 * mr) + (blk * 4 + (e % 16) / 4) * mr * 4 - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(dd)), ""), "") - LLVMBuildStore(b, w, dp) - } - } - } - var rNext = LLVMBuildAdd(b, rPhi, te.types->ConstI64(1ul), "iq2xx.rn") - var more = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, rNext, te.types->ConstI64(uint64(mr)), "iq2xx.more") - LLVMBuildCondBr(b, more, head, cont) - var loopVals <- [rNext] - var loopBlocks <- [head] - LLVMAddIncoming(rPhi, loopVals, loopBlocks) - LLVMPositionBuilderAtEnd(b, cont) -} // One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) // or the byte-expanded signed panel (33; the gemv gathers it per superblock, the tile gets the @@ -1819,18 +1698,10 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first var wbase = te.wg var wb : LLVMOpaqueValue? + var wbP : LLVMOpaqueValue? if (gather) { - if (te.kq == 33) { - emit_iq3s_gather(te, sbi) - } elif (te.kq == 23) { - emit_iq2s_gather(te, sbi) - } elif (te.kq == 24) { - emit_iq2xs_gather(te, sbi) - } elif (te.kq == 25) { - emit_iq2xxs_gather(te, sbi) - } else { - emit_iq3xxs_gather(te, sbi) - } + wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr) * uint64(kq_gather_stride(te.kq))), "kq.wb") + emit_kq_gather(te, sbi) wbase = te.iq3s_panel wb = te.types->ConstI64(0ul) } else { @@ -1872,6 +1743,12 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var a[i] = LLVMConstNull(te.vni32) a1[i] = LLVMConstNull(te.vni32) } + var signCol : LLVMOpaqueValue? [2] // sign="vec": this block's sign bytes, rows qd*rv.. at stride 4, one column load per vector row-band + if (gather && te.signVec) { + for (qd in range(rq)) { + signCol[qd] = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(kq_sign_col(te.kq, blk)) * uint64(mr * 4) + uint64(qd * w8)), ""), "sgc{blk}_{qd}") + } + } for (j in range(4)) { for (qd in range(rq)) { var noff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 16 + j * 4) * mr + qd * w8)), "") @@ -1881,6 +1758,10 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var var hoff = LLVMBuildAdd(b, noff, te.types->ConstI64(uint64(128 * mr)), "") wlo = load_vec(te, te.vwi8, wbase, noff, "wlo{blk}_{j * rq + qd}") whi = load_vec(te, te.vwi8, wbase, hoff, "whi{blk}_{j * rq + qd}") + if (signCol[qd] != null) { + wlo = apply_sign_col(te, wlo, signCol[qd], j / 2, j % 2, "wlo{blk}_{j * rq + qd}s") + whi = apply_sign_col(te, whi, signCol[qd], 2 + j / 2, j % 2, "whi{blk}_{j * rq + qd}s") + } } else { var nv = load_vec(te, te.vwi8, te.wg, noff, "nv{blk}_{j * rq + qd}") var loIdx = LLVMBuildAnd(b, nv, splat_i8w(te, 15), "nib.lo") @@ -2133,6 +2014,7 @@ def private setup_tile_emit(var te : TileEmit; var gc : LlvmCodeCtx; p : TilePer te.kgroup = (te.mx4 || te.q51 || te.kq != 0) ? 4 : rt.kgroup // the perm's bias applies to the Q8 weight plane only — mx4/kq/q51 planes never see the biased repack te.bias = (te.mx4 || te.q51 || te.kq != 0) ? 0 : p.bias + te.signVec = p.sign == "vec" te.width = (te.dotKind == DOT_SDOT || te.dotKind == DOT_SMMLA) ? 128 : p.width te.rv = te.width / 32 te.rq = rt.interleave / te.rv @@ -3050,7 +2932,7 @@ def private is_kq_gemv_signature(fn : Function const?) : bool { def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { if (!is_kq_gemv_signature(gc.fn)) return false let p0 = parse_perm(gc) - if (perm_declines(gc, p0)) return false + if (perm_declines(gc, p0) || sign_vec_declines(p0, fmt)) return false let p = companion_perm(p0) var te = TileEmit(kq = fmt) @@ -3200,7 +3082,7 @@ def private q51_gemv_gen(var gc : LlvmCodeCtx) : bool { def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { if (!is_tile_signature(gc.fn)) return false let p0 = parse_perm(gc) - if (perm_declines(gc, p0)) return false + if (perm_declines(gc, p0) || sign_vec_declines(p0, fmt)) return false let p = companion_perm(p0) var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // only the k5/k6 tiles get the byte-expanded panel diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 27f4e2932d..fb3ecdd018 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -690,6 +690,10 @@ def iq3sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(mr = 8, sign = "vec"), + tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq3sq8_gemv_gen", gen = "dasllama_gemm_gen::iq3s_gemv"), tune_companion(fn = "iq3sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3s_tile", @@ -969,6 +973,10 @@ def iq2sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(mr = 8, sign = "vec"), + tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq2sq8_gemv_gen", gen = "dasllama_gemm_gen::iq2s_gemv"), tune_companion(fn = "iq2sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2s_tile", diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index bbe9fb84cd..d651d8c0d5 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -775,6 +775,9 @@ vector sign mask replacing GPR sign math + abs), a `vpdpbssd` seat for AVX-VNNI-INT8 hosts, a repack-baked parity sign byte. Done = each of the five at or past the reference row at one thread, crowned by the probe; plan and fact base: `plans/kernel_parity_pass.md`. + 2026-09-01: `gather="reg"` measured 1.85x SLOWER (insertelement chains, 5x the code) and was + dropped; `sign="vec"` landed for iq3s (7732 us = 1.34x the reference) and iq2s (7076 us = + 0.72x). iq3xxs/iq2xs/iq2xxs join once the repack bakes their sign bytes into the plane. 62. **IQ3_S Metal decode: the ~140 GB/s compose ceiling (tg 0.95x).** Eight GEMV forms raced at n=2048 d=8192 - gather placement x3, gather deleted, signs deleted, llama.cpp's exact diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 1a595db3a5..daca3c1a7a 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -44,9 +44,9 @@ proven fact per push; research before any kernel edit (the two memos below). |---|---|---|---|---|---| | k4 | 1969 (8.6 ns/sb) | 2421 | 1.23x | 120.9 GFLOP/s | 71.1 GFLOP/s (1.70x) | | k2 | 2017 (8.8 ns/sb) | 2132 | 1.06x | - | - | -| iq3s | 11406 (49.7 ns/sb) | 10340 | 0.91x | - | - | +| iq3s | 11406 (49.7 ns/sb); sign=vec 7732 (33.7) | 10340 | 0.91x -> 1.34x | - | - | | iq3xxs | 11578 (50.5) | 6590 | 0.57x | - | - | -| iq2s | 11732 (51.1) | 5074 | 0.43x | - | - | +| iq2s | 11732 (51.1); sign=vec 7076 (30.8) | 5074 | 0.43x -> 0.72x | - | - | | iq2xs | 11490 (50.1) | 5386 | 0.47x | - | - | | iq2xxs | 11061 (48.2) | 5124 | 0.46x | - | - | @@ -86,9 +86,12 @@ bandwidth; on the 1B vehicles it shows, on a 27B it mostly does not - but the ke ## 4. Work queue (one item at a time; a row is done at >= 1.0x on rig 2's shape AND the vehicle) CPU decode (gap 2): -1. `gather="reg"` spelling on the shared gather emitter - one change, five formats; race in rig 1. -2. `psign="mask"`; 3. the IQ2_XXS / IQ2_XS load waste; 4. repack-baked parity sign; 5. retro - audit of IQ4_XS/Q3_K per followup 60 (already >= 1.0x; low priority). +1. DONE-KILLED `gather="reg"` (measured 1.85x slower - see the ledger). +2. DONE for iq3s/iq2s: `sign="vec"` (the plane-resident sign column, one masked negate per vector). +3. Repack-baked sign bytes for iq3xxs/iq2xs/iq2xxs (the memo's tier 4) so they join `sign="vec"`; + PACK_VERSION bump. 4. The iq2 word-pair index work - a u64 grid entry decoded as two dwords is + twice llama.cpp's index work per 8 weights (the remaining iq2s gap). 5. retro audit of + IQ4_XS/Q3_K per followup 60 (already >= 1.0x; low priority). Vulkan pp (gap 1): 1. Budget split (measurement, no kernel edit) - is the GEMM the 30%? 2. `shAscales`-style scale @@ -98,5 +101,15 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. ## 5. Ledger +- 2026-09-01: CPU item 1 measured and killed - `gather="reg"` (compose the weight vector with + insertelement, no panel) ran iq3s decode at 90.6 ns/sb against the panel's 49.1; the emitted IR + had 1025 insertelement chains and a 9904-instruction straight-line body against the panel loop's + 1893 (every load unique, so CSE was not the problem). The panel round trip is not the bottleneck. +- 2026-09-01: CPU item 2 landed as `sign="vec"` on iq3s and iq2s, the two formats whose sign bytes + sit in the plane at row stride 4: the panel holds raw grid words, one 32-byte column load per + block, a constant shuffle + and + cmpeq builds the byte mask, xor + sub negates the vector. + One thread, m=4096 k=14336: iq3s 11581 -> 7732 us (1.34x of the reference's 10340), iq2s 12413 + -> 7076 us (0.72x of 5074, was 0.41x). Every variant bit-exact in gen_tune_probe TEST mode. The + five gather emitters collapsed into one gather over per-format decode functions (-110 lines). - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. diff --git a/plans/kernel_parity_research_vk.md b/plans/kernel_parity_research_vk.md index ae8b68c209..53a0442fa3 100644 --- a/plans/kernel_parity_research_vk.md +++ b/plans/kernel_parity_research_vk.md @@ -10,22 +10,22 @@ One environment fact that shapes the whole comparison: the local llama.cpp Vulka reports `GL_NV_cooperative_matrix_decode_vector not supported by glslc` (`C:\Users\Boris\.claude\jobs\7f9893d9\tmp\tbo_build_vk.log`). llama.cpp therefore runs the scalar (one element per callback) decode path here, the same as ours. The comparison is -apples to apples today. It stops being so the moment that glslc is upgraded — see delta 7. +apples to apples today. It stops being so the moment that glslc is upgraded - see delta 7. --- ## 1. llama.cpp's cm2 GEMM, in ten facts -**F1 — One shader, six pipelines per quant type, three tile geometries.** +**F1 - One shader, six pipelines per quant type, three tile geometries.** Geometry arrives as spec constants `{BLOCK_SIZE, BM, BN, BK, enable_smaller_matrices}` (`D:\Work\llama.cpp\ggml\src\ggml-vulkan\vulkan-shaders\mul_mm_cm2.comp:31-39`). On coopmat2 the values are large `{256, 128, 256, 64, 1}`, medium `{256, 128, 128, 64, 1}`, small -`{256, 32, 64, 128, 0}` — and they are byte-identical for legacy quants and for K-quants +`{256, 32, 64, 128, 0}` - and they are byte-identical for legacy quants and for K-quants (`D:\Work\llama.cpp\ggml\src\ggml-vulkan\ggml-vulkan.cpp:4263-4277`). Each is stamped twice, unaligned and aligned, giving six (`ggml-vulkan.cpp:4664-4671`). There is **no per-format tile tuning** on the cm2 path. -**F2 — Operands: f16 times f16 into an f16 accumulator, stored as f32.** +**F2 - Operands: f16 times f16 into an f16 accumulator, stored as f32.** `mat_a` is `BM x BK` of the quantized weight, `mat_b` is `BK x BN` of the activation, both `MAT_TYPE = FLOAT_TYPE = float16_t` on coopmat2 (`mul_mm_cm2.comp:98-102`, `vulkan-shaders-gen.cpp:492-529`). `coopMatMulAdd` accumulates into `ACC_TYPE`, which is @@ -35,28 +35,28 @@ admitted to the cm2 path if it advertises fp16 A and B with both fp16 and fp32 a workgroup scope and flexible dimensions (`ggml-vulkan.cpp:6806-6838, 6863-6869`). Result is converted to `D_TYPE = float` for the store (`mul_mm_cm2.comp:493-495`). -**F3 — There is no integer activation path on coopmat2.** +**F3 - There is no integer activation path on coopmat2.** An f32 `src1` on a coopmat2 device is unconditionally marked non-contiguous (`ggml-vulkan.cpp:9276-9278`), which forces `quantize_y` false (`ggml-vulkan.cpp:9285`), and the q8_1 integer-dot `mul_mmq.comp` shader is never even compiled for coopmat2 (`vulkan-shaders-gen.cpp:626`). Activations are always f16 here. -**F4 — The f32-to-f16 activation conversion is one dispatch, cached across sibling matmuls.** +**F4 - The f32-to-f16 activation conversion is one dispatch, cached across sibling matmuls.** It runs through `ggml_vk_cpy_to_contiguous` into the `prealloc_y` scratch, but only when the last conversion did not already produce the same tensor with the same pipeline (`ggml-vulkan.cpp:9441-9453`, guarded by `prealloc_y_last_pipeline_used` / `prealloc_y_last_tensor_used`). Q, K and V therefore pay one conversion between them. -**F5 — Weights are decoded inside the driver's block load; nothing is pre-dequantized.** +**F5 - Weights are decoded inside the driver's block load; nothing is pre-dequantized.** `coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(...), dequantFuncA)` (`mul_mm_cm2.comp:381, 468`). The tensor layout carries the quant block size (`setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K)`, `mul_mm_cm2.comp:315-318`) so the driver resolves the block pointer and hands the callback a `buffer_reference` to it. The weight stays in ggml's on-disk block layout: no repack, no staging pass, no extra VRAM. -**F6 — Almost nothing is staged in shared memory, and the one exception is Q4_K/Q5_K scales.** +**F6 - Almost nothing is staged in shared memory, and the one exception is Q4_K/Q5_K scales.** `mul_mm_cm2.comp` declares only `row_ids` and `ballots_sh`, both under `MUL_MAT_ID` -(`mul_mm_cm2.comp:108, 116`) — contrast the non-cm2 shader, which stages both operands +(`mul_mm_cm2.comp:108, 116`) - contrast the non-cm2 shader, which stages both operands (`mul_mm.comp:130-131`). The exceptions are the iq codebooks (`types.glsl:918, 1031, 1201, 1501, 1573, 1683, 1813`, copied by `init_iq_shmem`) and a **Q4_K/Q5_K-only scale hoist**: `fetch_scalesQ4_K` reads the whole 16-byte scale head of a @@ -68,16 +68,16 @@ no shifting at all (`:537-540`). The comment states the intent outright: "we dec worth of scales into shared memory and then process the whole tile using those scales" (`:435-440`). -**F7 — Two tensor loads, one multiply-add, zero barriers per k-step.** +**F7 - Two tensor loads, one multiply-add, zero barriers per k-step.** The inner body is exactly `coopMatLoadTensorNV(mat_a, ...)`, `coopMatLoadTensorNV(mat_b, ...)`, `sum = coopMatMulAdd(mat_a, mat_b, sum)` (`mul_mm_cm2.comp:464-473`). The only barriers in the k loop are the two inside `store_scalesQ4_K` (`dequant_funcs_cm2.glsl:486, 522`), which fire once -per `QUANT_K = 256` elements of k — that is once per four `BK = 64` steps — and only for +per `QUANT_K = 256` elements of k - that is once per four `BK = 64` steps - and only for Q4_K/Q5_K. Every other format runs the whole k loop barrier-free. The scale fetch and store are explicitly pipelined: the next block's scales are fetched while the current block's are being consumed (`mul_mm_cm2.comp:363-374`). -**F8 — A clamp-free fast path, entered by construction rather than by luck.** +**F8 - A clamp-free fast path, entered by construction rather than by luck.** The fast path requires a whole M tile, a whole N tile against `padded_N`, `start_k % 256 == 0`, `end_k % BK == 0` and `stride_b % 8 == 0` (`mul_mm_cm2.comp:344-349`). Strides are then masked `&= ~7` purely to make 16-byte alignment provable to the driver's address analysis @@ -86,12 +86,12 @@ The fast path requires a whole M tile, a whole N tile against `padded_N`, `start token tile takes the unclamped path. The unaligned fallback uses clamp-Constant layouts and is marked `[[dont_unroll]]` (`mul_mm_cm2.comp:607`). -**F9 — The tail N tile gets a narrower accumulator.** +**F9 - The tail N tile gets a narrower accumulator.** `enable_smaller_matrices` makes the final column tile allocate `BN/2` or `BN/4` accumulators and loop over that width (`mul_mm_cm2.comp:36-38, 367-409, 410-452`), so a partial token column does not carry a full-width accumulator through the whole k loop. -**F10 — Split-k with a separate vectorised reduce.** +**F10 - Split-k with a separate vectorised reduce.** Split fires when `k >= 2048` and the tile grid fills less than half the SMs; the factor is `shader_core_count / tiles`, capped at 8, with the chunk rounded to 256 and splits shed if the last chunk would be empty (`ggml-vulkan.cpp:8795-8826`). Partials land in `prealloc_split_k` and a @@ -100,7 +100,7 @@ last chunk would be empty (`ggml-vulkan.cpp:8795-8826`). Partials land in `preal **Worked pick for a 1B prefill shape on this box** (M = 2048 weight rows, N = 512 tokens, K = 2048, 36 SMs): `tiles_m = 16 * 4 = 64 > 36`, so `prefer_large` is true and the aligned large -pipeline is chosen — BM 128, BN 256, BK 64, 256 invocations +pipeline is chosen - BM 128, BN 256, BK 64, 256 invocations (`ggml-vulkan.cpp:8839-8863`, `9309-9312`). Split-k: 32 tiles, neither `<= 18` nor `<= 24`, so `split_k = 1` (`ggml-vulkan.cpp:8795-8807`). Grid is 16 by 2 by 1 workgroups (`ggml-vulkan.cpp:8896`, `8221-8229`). @@ -114,55 +114,55 @@ subgroup-ballot prefix scan and stores through a per-element callback ## 2. Ours, in the same terms -**O1 — Same three tile geometries, same workgroup size, same accumulator type.** +**O1 - Same three tile geometries, same workgroup size, same accumulator type.** `KqCm2BatchT::run` is `[spirv_kernel(local_size_x = 256)]` (`D:\Work\daScript-iquant\modules\dasLLAMA\dasllama\dasllama_vulkan_classes.das:4825`). A is `coopmatWgA_f16_128x64` (`:4888`), so BM 128 and BK 64 are fixed; BN is a template constant of 256, 128 or 32 with matching B and accumulator types (`:4809`, `:5336-5361` for q8 and the same four typedefs repeated per format through `:5713`). The accumulator is -`coopmatWgAcc_f16_128xBN` — f16, like llama.cpp's default — clamped to the f16 range and widened +`coopmatWgAcc_f16_128xBN` - f16, like llama.cpp's default - clamped to the f16 range and widened to f32 only for the store (`:4923-4926`). -**O2 — Same tile pick, arrived at independently and landing in the same place.** +**O2 - Same tile pick, arrived at independently and landing in the same place.** `cm2_tile_cols` compares occupied against allocated wave slots cross-multiplied, ties to large, with `cnt <= 64` forcing small and `cnt <= 128` forcing medium (`D:\Work\daScript-iquant\modules\dasLLAMA\dasllama\dasllama_vulkan_common.das:129-151`). For d = 2048, cnt = 512, 36 cores it returns 256, the same large tile llama.cpp picks. -**O3 — Same split-k rule, near-identical constants.** +**O3 - Same split-k rule, near-identical constants.** `cm2_split_k` gates on `n >= 2048`, `d >= 128`, `cnt >= tc`, computes `cores / tiles` capped at 8, rounds the chunk to 256 and sheds stranded splits (`dasllama_vulkan_common.das:155-179`). The in-kernel plane decomposition is `dasllama_vulkan_classes.das:4832-4846`; the reduce is a separate class (`:5722-5751`), encoded at `dasllama_vulkan_prefill.das:355-366`. -**O4 — Same decode-in-load mechanism.** +**O4 - Same decode-in-load mechanism.** `coopmatLoadTensorDecode(a, wq, wblk0, fla, m0, 128u, k, 64u, self.decode)` (`dasllama_vulkan_classes.das:4911`), with the block size on the layout (`tensorLayoutSetBlockSize(fla, 1u, BLKW)`, `:4896`). Weights are never materialised in f16. -**O5 — Same fast/edge split and the same stride-masking trick.** +**O5 - Same fast/edge split and the same stride-masking trick.** Fast path when `m0 + 128 <= pa.d`, the token column is whole or the small tile is stamped, and `pa.n % 64 == 0` (`:4893`); strides masked `& ~7u` on the B and output layouts (`:4902, 4906`). Edge path uses clamp-Constant layouts (`:4950-5005`). -**O6 — Same k-loop shape, one degree more unrolling, zero barriers.** +**O6 - Same k-loop shape, one degree more unrolling, zero barriers.** Two loads and one `coopmatMulAdd` per 64-wide k step, the outer loop unrolled 8 deep for 512 k per iteration with a scalar drain (`:4907-4922`). The kernel contains exactly one `barrier()`, before the loop, to publish `wg_blk0` and any staged codebook (`:4887`). llama.cpp unrolls 4 deep at `QUANT_K = 256` (`mul_mm_cm2.comp:341-342`). -**O7 — The activation f16 conversion is likewise one dispatch per sibling group.** +**O7 - The activation f16 conversion is likewise one dispatch per sibling group.** `f16cvt_cls` feeds q/k/v (`dasllama_vulkan_prefill.das:518-524`) and gate/up (`:562-568`); `actf16_cls` fuses the SwiGLU epilogue with the conversion feeding down (`:569-576`). All of it is recorded into the measured command buffer. This is at least as good as llama.cpp's caching, and the fused activation is strictly better. -**O8 — Difference: the weight scale plane is a second SSBO, addressed by hand, per element.** +**O8 - Difference: the weight scale plane is a second SSBO, addressed by hand, per element.** `wq` (quants) is binding 0 and `ws` (scales) is binding 1 (`dasllama_vulkan_classes.das:4804-4805`). Every decode callback computes its own scale address `(wg_blk0 + bc.x * (pa.n >> 8u) + bc.y) * 5u` and then extracts the sub-block scale with a 32-bit -load and a **runtime-variable shift** — per decoded element. Q4_K is the worst case, three +load and a **runtime-variable shift** - per decoded element. Q4_K is the worst case, three separate `ws` dwords at a 20-byte stride plus three variable shifts (`:5034-5038`). Q6_K (`:5058-5062`), Q3_K (`:5143-5147`), Q2_K (`:5164-5167`), IQ4_XS (`:5122-5125`), IQ3_S (`:5190-5193`), IQ3_XXS (`:5223-5226`), IQ2_S (`:5269-5273`) and IQ2_XS @@ -170,37 +170,37 @@ separate `ws` dwords at a 20-byte stride plus three variable shifts driver already resolved, and for Q4_K/Q5_K does the extraction eight times per 256 elements instead of 256 times (F6). -**O9 — Difference: no shared-memory scale hoist for any format.** +**O9 - Difference: no shared-memory scale hoist for any format.** The only workgroup memory declared is `wg_blk0` (4 bytes) plus one gated iq codebook of 32 bytes to 8 KB (`dasllama_vulkan_classes.das:4808-4823`). There is no `shAscales` equivalent. -**O10 — Difference: the codebooks are computed per workgroup, not copied.** +**O10 - Difference: the codebooks are computed per workgroup, not copied.** `iq3s_gridc[gi] = iq3s_grid_word(int(gi))` and its siblings synthesise the table with arithmetic at kernel entry (`:4861-4886`), where llama.cpp copies from a constant array (`types.glsl:1204-1213`). -**O11 — Difference: a per-workgroup prologue that llama.cpp does not have.** -Each workgroup reads a region record from a `sched` SSBO — five scalar loads -(`dasllama_vulkan_classes.das:3972-3976`) — broadcasts `wg_blk0` through workgroup memory +**O11 - Difference: a per-workgroup prologue that llama.cpp does not have.** +Each workgroup reads a region record from a `sched` SSBO - five scalar loads +(`dasllama_vulkan_classes.das:3972-3976`) - broadcasts `wg_blk0` through workgroup memory (`:4849-4851`) and barriers (`:4887`). llama.cpp derives everything from `gl_WorkGroupID` with no loads and no leading barrier. Ours buys MoE region lists with this; llama.cpp pays for the same capability with a separate `MUL_MAT_ID` shader. -**O12 — Difference: no N padding and no narrow-tail accumulator.** +**O12 - Difference: no N padding and no narrow-tail accumulator.** The activation plane is sized to `cnt`, not to a tile multiple, for the large and medium tiles (`:4901`), so a partial last token column drops to the fully clamped edge path (`:4893`). Only the small tile rounds its dimension up to 32 and keeps the unclamped load. There is no `enable_smaller_matrices` analogue. At exactly 512 tokens with BN 256 this costs nothing; at 500 tokens it costs the last tile. -**O13 — Grid: one-dimensional.** +**O13 - Grid: one-dimensional.** `vkCmdDispatch(raw, groups, 1u, 1u)` (`dasllama_vulkan_common.das:2469-2478`) with `groups = ceil(rows/BN) * ceil(d/128) * nsplit` (`dasllama_vulkan_prefill.das:1089-1106`). Push constants are 16 bytes, `{n, d, map_off, ksplit}` (`dasllama_vulkan_classes.das:2326-2331`); everything else rides the SSBO region record. llama.cpp pushes a 17-word block with the full geometry (`mul_mm_cm2.comp:41-70`). -**O14 — The measured ratios are NOT a uniform band, and the followup says otherwise.** +**O14 - The measured ratios are NOT a uniform band, and the followup says otherwise.** `followup_vulkan.md:540-546` (entry 34) states "~0.67-0.70x" and "every sb-format cm2 tile lands in the same band" and "the gap is the shared batch-GEMM tier, not any one format's decode". The per-format tables in @@ -213,12 +213,12 @@ b10660 Vulkan at `-ngl 99` on 1B-class vehicles, say: | Q2_K | 14544.0 / 16752.7 | 0.87x | `:774` | | Q3_K | 14031 / 17509 | 0.80x | `:1088` | | IQ4_NL | 15027.4 / 19177.8 | 0.78x | `:831` | -| IQ2_XXS | — | 0.78x | `:554` | +| IQ2_XXS | - | 0.78x | `:554` | | IQ2_XS | 14023.3 / 18320.0 | 0.77x | `:606` | | IQ3_S | 12539.6 / 17865 | 0.70x | `:1020` | | IQ2_S | 12099.3 / 17377.5 | 0.70x | `:702` | | IQ3_XXS | 12225.7 / 17807.7 | 0.69x | `:918` | -| Q4_K (the control) | — | 0.67x | `:987` | +| Q4_K (the control) | - | 0.67x | `:987` | That is a 1.34x spread, not a band. Entry 34 was written on 2026-08-30 (`followup_vulkan.md:543`) when only k4 and iq3s had Vulkan rows; six formats landed after it @@ -226,12 +226,12 @@ and none updated it. The document contradicts itself at `HOW_TO_ADD_A_FORMAT.md: ("0.77x - above the 0.70x tier class") and `:753-754` ("0.87x - above the 0.69-0.78 tier class"). -**O15 — The "shared tier" diagnosis is refuted by our own Q8 rows.** +**O15 - The "shared tier" diagnosis is refuted by our own Q8 rows.** Q8_0 on the same tile, the same chain, the same box measures 3B pp 7737.2 = 100.6% of the reference exe and tinyllama pp 20986 = about 103.5% (`followup_vulkan.md:211-213`). If the deficit were the shared batch-GEMM tier, Q8 would show it. And the kernel-rate probes already point at the decode: Q8 cm2 clocks 62.2/64.7 TF/s on gate/down (`followup_vulkan.md:176-178`) while Q4_K -clocks 35.8-38.2 TF/s, "about 70% of Q8-cm2's rate — the nibble+scale extraction" +clocks 35.8-38.2 TF/s, "about 70% of Q8-cm2's rate - the nibble+scale extraction" (`followup_vulkan.md:223-224`); Q6_K sat at 9.3-13.4 TF/s until one byte4 dynamic select was respelled, after which it hit 32.9 TF/s (`followup_vulkan.md:229-234`). @@ -246,12 +246,12 @@ Shape assumed throughout: dim 2048-3072, hidden 8192, 512 tokens, so the large t (BM 128, BN 256, BK 64) and split-k off on the wide GEMMs, split-k 2 on the narrow k/v projections. -### Delta 1 — Hoist the per-(row, sub-block) scale into workgroup memory, llama.cpp's `shAscales` form. Estimated payoff: 10-20% on Q4_K/Q5_K, less on the rest. +### Delta 1 - Hoist the per-(row, sub-block) scale into workgroup memory, llama.cpp's `shAscales` form. Estimated payoff: 10-20% on Q4_K/Q5_K, less on the rest. Ours re-derives the scale for every decoded element: a 32-bit load from a second SSBO plus a runtime-variable shift plus a mask plus a float convert, three times over for Q4_K (`dasllama_vulkan_classes.das:5034-5038`). llama.cpp does that arithmetic once per (row, -sub-block) — 8 times per 256 elements per row — into `shAscales`, and the element decode is then +sub-block) - 8 times per 256 elements per row - into `shAscales`, and the element decode is then a single `vec2` read with no shifting (`dequant_funcs_cm2.glsl:484-523, 537-540`). That is a 32-fold reduction in scale-extraction ALU on the format that is currently our worst. @@ -271,13 +271,13 @@ and its `flat` variant is "current compose w/o scale reads" q6k gate and down shapes it already pins (9728x2560x512 and 2560x9728x512, `vk_gemm_probe.das:1287-1288`) and read the `nil` to `flat` to shipped ladder. The `flat`-to- shipped gap is the entire scale-plane cost in TF/s. If it is under 5%, delta 1 is dead. Then add -a k4 twin of that bisect — the file has no k4 scale-plane arm today, only a k4 rate arm +a k4 twin of that bisect - the file has no k4 scale-plane arm today, only a k4 rate arm (`vk_gemm_probe.das:357`). Caveat: arm `DAS_TUNE_MANIFEST` or the checked-in `performance/last_known_good_sidecar.json` first, or a bare probe run re-execs into a full retune (`ARCHITECTURE_MEASUREMENT.md:53-56`). -### Delta 2 — Fold the scale plane back into the quant block, so the decode touches one buffer. Estimated payoff: 5-15%, uniform across formats. +### Delta 2 - Fold the scale plane back into the quant block, so the decode touches one buffer. Estimated payoff: 5-15%, uniform across formats. llama.cpp's callback gets a `buffer_reference` the driver already resolved and reads d, dmin and all 12 scale bytes as one 16-byte aligned `uvec4` (`dequant_funcs_cm2.glsl:429-431, 461`). Ours @@ -296,7 +296,7 @@ k4 fixture whose `ws` contents are interleaved into the `wq` plane and read thro block pointer. Compare TF/s on one shape (d 2048, K 2048, cnt 512, large tile, split off via `DASLLAMA_CM2_SPLITK=1`). Kill it if the interleaved arm is within 3%. -### Delta 3 — Split the end-to-end budget before spending anything on the kernel. Estimated payoff: none directly; it decides whether deltas 1, 2 and 4-8 are worth doing at all. +### Delta 3 - Split the end-to-end budget before spending anything on the kernel. Estimated payoff: none directly; it decides whether deltas 1, 2 and 4-8 are worth doing at all. Every ratio in the table above is end-to-end pp512 tokens per second, not GEMM time. An end-to-end 0.67x does not prove the GEMM is 0.67x. The chain around it carries 452 nodes and 367 @@ -306,7 +306,7 @@ barriers per window on the 3B (`followup_vulkan.md:246-248`), plus the f16 conve **Measurement.** Two halves, both cheap. On llama.cpp: `set GGML_VK_PERF_LOGGER=1` and run the same model and prompt through -`llama-bench -p 512 -n 0 -ngl 99`. It prints one line per op keyed by shape — +`llama-bench -p 512 -n 0 -ngl 99`. It prints one line per op keyed by shape - `MUL_MAT q4_K m=2048 n=512 k=2048: N x T us = T us (G GFLOPS/s)` (`ggml-vulkan.cpp:2278-2297, 2240-2254`). On ours: the prefill already timestamps every role (`pfq_ts`, `dasllama_vulkan_prefill.das:391`, @@ -316,20 +316,20 @@ If our MUL_MAT total is at parity and the wall clock is not, the deficit is outs deltas 1-2 are the wrong target. If our MUL_MAT total is 0.67x, the deficit is in the kernel and delta 1 leads. -### Delta 4 — Pad the activation plane's N to the tile width and keep the fast path. Estimated payoff: zero at exactly 512 tokens, up to about `1/ceil(N/BN)` of the GEMM at other token counts. +### Delta 4 - Pad the activation plane's N to the tile width and keep the fast path. Estimated payoff: zero at exactly 512 tokens, up to about `1/ceil(N/BN)` of the GEMM at other token counts. llama.cpp pads B's N to the tile width in the prealloc buffer specifically so the last token column still takes the unclamped path (`ggml-vulkan.cpp:9319`, gate at `mul_mm_cm2.comp:345`). Ours sizes the plane to `cnt` and drops the partial column onto the fully clamped edge path (`dasllama_vulkan_classes.das:4893, 4901`). At 512 tokens with BN 256 there is -no partial column, so this is invisible in the pp512 board — and visible in every real prompt +no partial column, so this is invisible in the pp512 board - and visible in every real prompt whose length is not a multiple of 256. **Measurement.** Run pp at 512 and at 500 tokens, ours and llama.cpp, same model. Our ratio should degrade at 500 and theirs should not. If both degrade equally, the edge path is not the cause. -### Delta 5 — Narrow the tail accumulator (`enable_smaller_matrices`). Estimated payoff: same trigger as delta 4, roughly half its size. +### Delta 5 - Narrow the tail accumulator (`enable_smaller_matrices`). Estimated payoff: same trigger as delta 4, roughly half its size. llama.cpp allocates a `BN/2` or `BN/4` accumulator for the tail column (`mul_mm_cm2.comp:36-38, 367, 410`), so a 64-token remainder does not drag a 256-wide @@ -339,7 +339,7 @@ refinement on top of it. **Measurement.** Only meaningful after delta 4 lands. Same 500-token comparison, with and without a narrowed tail stamp. -### Delta 6 — Reduce the inner unroll from 8 to 4. Estimated payoff: 0 to 5%, sign unknown. +### Delta 6 - Reduce the inner unroll from 8 to 4. Estimated payoff: 0 to 5%, sign unknown. Ours unrolls 8 deep, 512 k per outer iteration (`dasllama_vulkan_classes.das:4909-4916`); llama.cpp unrolls 4 at `QUANT_K = 256` (`mul_mm_cm2.comp:341-342`). With an f16 128x256 @@ -353,13 +353,13 @@ speculative. all three on one shape through the probe's shape arms. Read the driver's register/occupancy report if Nsight is attached; otherwise the timing alone decides. -### Delta 7 — No four-wide decode callback. Estimated payoff: zero today, large the day glslc is upgraded. +### Delta 7 - No four-wide decode callback. Estimated payoff: zero today, large the day glslc is upgraded. llama.cpp ships a `_v` twin of every decode function returning `f16vec4` (`dequant_funcs_cm2.glsl:576-623` for Q4_K, `:778-805` for Q6_K, `:1002-1025` for IQ2_XS, and so on for every format), wired through `GGML_VULKAN_COOPMAT2_DECODE_VECTOR` (`mul_mm_cm2.comp:82-86`, `vulkan-shaders-gen.cpp:480-484`). The local glslc cannot build it -(`tbo_build_vk.log`), so llama.cpp runs scalar here — but the moment that toolchain moves, +(`tbo_build_vk.log`), so llama.cpp runs scalar here - but the moment that toolchain moves, llama.cpp's callback count drops fourfold and every delta above gets 4x harder to close. Our `[spirv_decode]` surface has no vec4 form. @@ -367,7 +367,7 @@ llama.cpp's callback count drops fourfold and every delta above gets 4x harder t supports the extension and re-run the same per-format board; the delta between the two llama.cpp builds is what we would owe. -### Delta 8 — The per-workgroup prologue and the computed codebooks. Estimated payoff: under 2% on the dense 1B shape; real on MoE grids. +### Delta 8 - The per-workgroup prologue and the computed codebooks. Estimated payoff: under 2% on the dense 1B shape; real on MoE grids. Five `sched` loads, a workgroup broadcast and a barrier before the first tile load (`dasllama_vulkan_classes.das:3972-3976, 4849-4851, 4887`), plus synthesising 256 to 2048 @@ -375,8 +375,8 @@ codebook words with arithmetic rather than copying them (`:4861-4886`). On a 32- dispatch this amortises away. On a MoE expert grid with many small dispatches it does not, which is the GEMV twin of the problem `followup_vulkan.md:548-557` (entry 35) already flags for tg. -**Measurement.** A variant that hardcodes the single-region dense case — region 0, workgroup base -0, no `sched` read — A/B against the shipped class on the same shape. If the dense arm gains +**Measurement.** A variant that hardcodes the single-region dense case - region 0, workgroup base +0, no `sched` read - A/B against the shipped class on the same shape. If the dense arm gains under 2%, leave it alone and route the effort to the MoE grids instead. **One correction to file regardless of the outcome.** `followup_vulkan.md:540-546` (entry 34) @@ -417,7 +417,7 @@ GPU backend, duplicates the op that many times into one graph, does a warmup com whole-graph submits until at least one second has elapsed (`:1569-1601`, `:1620-1636`). **What it gives us as a reference row:** a per-kernel GFLOPS number for llama.cpp's cm2 GEMM at a -fixed shape, isolated from the rest of a forward pass, on the same device and driver — the exact +fixed shape, isolated from the rest of a forward pass, on the same device and driver - the exact counterpart to `vk_gemm_probe.das`'s per-shape TF/s rows. Because the graph repeats one node, the f32-to-f16 activation conversion is converted once and reused across the duplicates (`ggml-vulkan.cpp:9441-9453`), so the reported GFLOPS is close to pure GEMM. @@ -433,7 +433,7 @@ for (int bs : {1, 2, 3, 4, 5, 8, 512}) ``` (`:10321-10323`) -so the available shape is **m 4096, n 512, k 14336** — a Llama-8B FFN shape, not a 1B/3B one. +so the available shape is **m 4096, n 512, k 14336** - a Llama-8B FFN shape, not a 1B/3B one. `-p` can only filter what is already in the list; it cannot create a 2048x512x2048 or 3072x512x8192 row. Getting our shape needs either a one-line addition to that loop (a local patch to a tree this task treats as read-only) or `--test-file` with a graph exported by From c6ce3c1e8284250084a747953ff885652714be34 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 00:52:44 -0700 Subject: [PATCH 003/123] kernel parity, CPU item 2b: sign="vec" reaches iq3xxs, iq2xs and iq2xxs - the sign column synthesized from the 7-bit codes The 8th sign bit is the code's parity (ksigns[i] == i | parity(i) << 7), so the sign-byte column the masked negate wants is seven vector ops on the aux column (iq3xxs, iq2xxs) or on the u16 word column (iq2xs) - no plane layout change, no image version bump, nothing for the disk kernels or the GPU tiers. One thread, m=4096 k=14336: iq3xxs 11308 -> 7882 us (reference 6590), iq2xxs 10967 -> 7832 (5124), iq2xs 11711 -> 9197 (5386). All 20 sign_vec variants ok in gen_tune_probe TEST mode. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 51 +++- .../dasLLAMA/dasllama/dasllama_math_gen.das | 12 + plans/kernel_parity_research_cpu.md | 250 +++++++++--------- 3 files changed, 179 insertions(+), 134 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index d5a665a2bb..418190d690 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -90,7 +90,7 @@ struct private TilePerm { mr : int = GEMM_REFERENCE_MR() gkstep : int = 1 dotPrim : string = "sdot" - sign : string = "scalar" // iq3s/iq2s gemv: "scalar" = signs applied per grid dword in the gather, "vec" = raw words in the panel, one masked negate per weight vector off the plane's sign column + sign : string = "scalar" // the grid formats' gemv: "scalar" = signs applied per grid dword in the gather, "vec" = raw words in the panel, one masked negate per weight vector off a sign-byte column width : int = 128 bias : int = 0 // 128 = bias128: grp bytes baked as w^0x80, plain vpdpbusd dots pipe : int = 0 // 1 = software-pipelined amx fold (slice K I4): double-buffered C @@ -1335,11 +1335,44 @@ def private signed_grid_word(var te : TileEmit; var gidx, sgn : LLVMOpaqueValue? return LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, LLVMConstInt(te.types.t_int32, 0x01010101ul, 0), ""), "") } -//! sign="vec" needs the 8-weight sign bytes plane-resident at row stride 4: iq3s and iq2s only -def private sign_vec_declines(p : TilePerm; fmt : int) : bool => p.sign == "vec" && fmt != 33 && fmt != 23 - -//! the grp column (4 bytes per row) holding block `blk`'s four sign bytes -def private kq_sign_col(kq, blk : int) : int => (kq == 33 ? 18 : 8) + blk +//! sign="vec": block `blk`'s sign bytes for rows qd*rv.., byte (r, l) at r*4 + l - the column +//! itself for iq3s/iq2s, synthesized from the 7-bit ksigns codes for the other three (the 8th +//! bit is the code's parity, ksigns[i] == i | parity(i) << 7) +def private kq_sign_bytes(var te : TileEmit; var wbP : LLVMOpaqueValue?; blk, qd : int; name : string) : LLVMOpaqueValue? { + let b = te.builder + let mr = te.interleave + let w8 = te.width / 8 + let colBytes = uint64(mr * 4) + if (te.kq == 33 || te.kq == 23) { + let col = (te.kq == 33 ? 18 : 8) + blk + return load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(col) * colBytes + uint64(qd * w8)), ""), name) + } + var codesCol : LLVMOpaqueValue? [2] // iq2xs: columns 2blk (l = 0, 1) and 2blk+1 (l = 2, 3) of u16 words; else one aux32 column + if (te.kq == 24) { + for (c in range(2)) { + codesCol[c] = load_vec(te, te.vni32, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(2 * blk + c) * colBytes + uint64(qd * w8)), ""), "sgw{blk}_{c}") + } + } else { + let col = te.kq == 34 ? 16 + blk : 2 * blk + 1 + codesCol[0] = load_vec(te, te.vni32, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(col) * colBytes + uint64(qd * w8)), ""), "sga{blk}") + } + var bytes = LLVMConstNull(te.vni32) + for (l in range(4)) { + var code : LLVMOpaqueValue? + if (te.kq == 24) { + var word = l % 2 == 0 ? LLVMBuildAnd(b, codesCol[l / 2], splat_i32n(te, 65535), "") : LLVMBuildLShr(b, codesCol[l / 2], splat_i32n(te, 16), "") + code = LLVMBuildLShr(b, word, splat_i32n(te, 9), "") + } else { + code = LLVMBuildAnd(b, l == 0 ? codesCol[0] : LLVMBuildLShr(b, codesCol[0], splat_i32n(te, 7 * l), ""), splat_i32n(te, 127), "") + } + var par = LLVMBuildXor(b, code, LLVMBuildLShr(b, code, splat_i32n(te, 4), ""), "") + par = LLVMBuildXor(b, par, LLVMBuildLShr(b, par, splat_i32n(te, 2), ""), "") + par = LLVMBuildXor(b, par, LLVMBuildLShr(b, par, splat_i32n(te, 1), ""), "") + var sb = LLVMBuildOr(b, code, LLVMBuildShl(b, LLVMBuildAnd(b, par, splat_i32n(te, 1), ""), splat_i32n(te, 7), ""), "") + bytes = LLVMBuildOr(b, bytes, l == 0 ? sb : LLVMBuildShl(b, sb, splat_i32n(te, 8 * l), ""), "") + } + return LLVMBuildBitCast(b, bytes, te.vwi8, name) +} //! one masked negate per weight vector: byte (r, k) of the mask is 0xFF when bit (h*4 + k) of row r's //! sign byte for group `l` is set - a constant shuffle broadcasts each row's byte to its four lanes @@ -1746,7 +1779,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var var signCol : LLVMOpaqueValue? [2] // sign="vec": this block's sign bytes, rows qd*rv.. at stride 4, one column load per vector row-band if (gather && te.signVec) { for (qd in range(rq)) { - signCol[qd] = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(kq_sign_col(te.kq, blk)) * uint64(mr * 4) + uint64(qd * w8)), ""), "sgc{blk}_{qd}") + signCol[qd] = kq_sign_bytes(te, wbP, blk, qd, "sgc{blk}_{qd}") } } for (j in range(4)) { @@ -2932,7 +2965,7 @@ def private is_kq_gemv_signature(fn : Function const?) : bool { def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { if (!is_kq_gemv_signature(gc.fn)) return false let p0 = parse_perm(gc) - if (perm_declines(gc, p0) || sign_vec_declines(p0, fmt)) return false + if (perm_declines(gc, p0)) return false let p = companion_perm(p0) var te = TileEmit(kq = fmt) @@ -3082,7 +3115,7 @@ def private q51_gemv_gen(var gc : LlvmCodeCtx) : bool { def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { if (!is_tile_signature(gc.fn)) return false let p0 = parse_perm(gc) - if (perm_declines(gc, p0) || sign_vec_declines(p0, fmt)) return false + if (perm_declines(gc, p0)) return false let p = companion_perm(p0) var te = TileEmit(kq = fmt, kqBytes = fmt == 5 || fmt == 6) // only the k5/k6 tiles get the byte-expanded panel diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index fb3ecdd018..7342a271cb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -769,6 +769,10 @@ def iq3xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(mr = 8, sign = "vec"), + tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq3xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq3xxs_gemv"), tune_companion(fn = "iq3xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3xxs_tile", @@ -1048,6 +1052,10 @@ def iq2xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(mr = 8, sign = "vec"), + tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq2xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xs_gemv"), tune_companion(fn = "iq2xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xs_tile", @@ -1121,6 +1129,10 @@ def iq2xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(mr = 8, sign = "vec"), + tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq2xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xxs_gemv"), tune_companion(fn = "iq2xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xxs_tile", diff --git a/plans/kernel_parity_research_cpu.md b/plans/kernel_parity_research_cpu.md index dcff040f4d..7f36a319f4 100644 --- a/plans/kernel_parity_research_cpu.md +++ b/plans/kernel_parity_research_cpu.md @@ -1,13 +1,13 @@ # CPU grid-quant dot: what llama.cpp does, what we do, and the gap -Read-only study of five codebook ("grid") formats — IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S — +Read-only study of five codebook ("grid") formats - IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S - plus Q2_K as the near-parity control, for the single-token decode shape (GEMV). Trees read: -- `D:\Work\llama.cpp` — `ggml/src/ggml-cpu/arch/x86/quants.c`, `arch/arm/quants.c`, +- `D:\Work\llama.cpp` - `ggml/src/ggml-cpu/arch/x86/quants.c`, `arch/arm/quants.c`, `quants.c`, `ggml-cpu.c`, `repack.cpp`, `llamafile/sgemm.cpp`, `ggml/src/ggml-common.h` -- `D:\Work\daScript-iquant` — `modules/dasLLAMA/dasllama/dasllama_gemm_gen.das`, +- `D:\Work\daScript-iquant` - `modules/dasLLAMA/dasllama/dasllama_gemm_gen.das`, `dasllama_math_gen.das`, `dasllama_repack.das`, `modules/dasLLVM/daslib/llvm_tune.das` Nothing was modified in either tree. @@ -30,14 +30,14 @@ ARM, with a cross-lane `vaddvq_s32` per 32-weight sub-block (arm quants.c:3678). (gemm_gen.das:2123-2141) sets `rv = width/32` and `rq = mr/rv`; one weight vector holds **4 bytes from each of `rv` different output rows**. `dot_lane` (gemm_gen.das:328-366) splats one dword-group of the activation across every lane and dots. For the seated x86 perm -(`width = 256, mr = 8`) one `<32 x i8>` weight vector = 8 rows × 4 weights; for AVX-512 -(`width = 512, mr = 16`) it is 16 rows × 4 weights; on NEON (`width = 128, mr = 8, rq = 2`) -it is two vectors of 4 rows × 4 weights. +(`width = 256, mr = 8`) one `<32 x i8>` weight vector = 8 rows x 4 weights; for AVX-512 +(`width = 512, mr = 16`) it is 16 rows x 4 weights; on NEON (`width = 128, mr = 8, rq = 2`) +it is two vectors of 4 rows x 4 weights. Consequences, both directions: - **We win the reduction.** Our accumulator lanes *are* the output rows. There is no - horizontal sum anywhere in the kernel — `emit_block_iq4xs` ends with one `sitofp` + one + horizontal sum anywhere in the kernel - `emit_block_iq4xs` ends with one `sitofp` + one FMA per superblock per row-lane (gemm_gen.das:1943-1953). llama.cpp's ARM kernels pay a `vaddvq_s32` per 32-weight block per row (arm:3678, 3845, 3911, 4021), i.e. 8 cross-lane reductions per superblock per row on a serial dependency chain. This is almost certainly @@ -45,11 +45,11 @@ Consequences, both directions: - **We cannot put signs on the activation.** The 8 lanes of one weight vector belong to 8 *different* rows with 8 *different* sign patterns, while the activation dword is shared across all lanes. Signs must go on the weights. This is a hard consequence of the layout, - not a choice, and it is the part of followup entry 61 that needs correcting (see §8). + not a choice, and it is the part of followup entry 61 that needs correcting (see sec.8). - **Grid decode cannot stream.** A grid word is 4 or 8 bytes of *one* row. In llama.cpp's layout, 8 consecutive grid words of one row *are* the next vector, so the composition `_mm256_set_epi32(grid[i7],...,grid[i0])` writes the register directly. In our layout a - weight vector needs one grid dword from each of 8 different rows — which is *also* exactly + weight vector needs one grid dword from each of 8 different rows - which is *also* exactly a `set_epi32` of 8 table reads. The composition is available to us; we just do not use it. Today we round-trip through an `alloca` panel instead. @@ -63,48 +63,48 @@ Consequences, both directions: by all threads, into `params->wdata` (ggml-cpu.c:1321-1355). For all six formats `vec_dot_type = GGML_TYPE_Q8_K` (ggml-cpu.c:298-366). - `nrows = 1` for all six (ggml-cpu.c:302, 338, 344, 350, 356, 362), so - `num_rows_per_vec_dot` is 1 (ggml-cpu.c:1435) — no 2-row MMLA fast path, unlike Q4_K/Q6_K + `num_rows_per_vec_dot` is 1 (ggml-cpu.c:1435) - no 2-row MMLA fast path, unlike Q4_K/Q6_K which get `nrows = 2` under `__ARM_FEATURE_MATMUL_INT8` (ggml-cpu.c:313-317, 331-335). - Work is chunked over rows and dispatched to `ggml_compute_forward_mul_mat_one_chunk` (ggml-cpu.c:1164), which calls `vec_dot` per row. - **`from_float` is NULL for all five grid formats** (ggml-cpu.c:337, 343, 348-349, 356, 362) - — these types are never produced on the fly, only read. + - these types are never produced on the fly, only read. **Repack / "extra buffer type": none of the five grid formats has one.** `ggml_repack_get_optimal_repack_type` (repack.cpp:4528) covers Q4_0, Q8_0, Q4_K, Q5_K, Q6_K, -MXFP4, IQ4_NL, and **Q2_K** — nothing else. So for IQ2_XXS/IQ2_XS/IQ2_S/IQ3_XXS/IQ3_S, +MXFP4, IQ4_NL, and **Q2_K** - nothing else. So for IQ2_XXS/IQ2_XS/IQ2_S/IQ3_XXS/IQ3_S, decode is *always* the per-row `vec_dot` above. **Q2_K is the exception and it matters for the control measurement.** repack.cpp:4627-4632 selects `q2_K_8x8_q8_K` when `ggml_cpu_has_avx512()` and `ne[1] % 8 == 0`; the GEMV entry is -`gemv` (repack.cpp:3976-3984) → `ggml_gemv_q2_K_8x8_q8_K`. -That is an 8-row-interleaved GEMV — *our* shape. There is also a RISC-V `q2_K_16x1_q8_K` +`gemv` (repack.cpp:3976-3984) -> `ggml_gemv_q2_K_8x8_q8_K`. +That is an 8-row-interleaved GEMV - *our* shape. There is also a RISC-V `q2_K_16x1_q8_K` (repack.cpp:4633-4642). So a Q2_K comparison on zen2 (no AVX-512) races `ggml_vec_dot_q2_K_q8_K`, and the same comparison on zen4 races the repacked 8x8 gemv. Worth keeping straight when reading the 1.02x/0.95x control numbers. `llamafile_sgemm` (ggml-cpu.c:1293-1318, 1364-1385) covers only F32, BF16, F16, Q8_0, Q4_0, -Q5_0, IQ4_NL (sgemm.cpp:3829, 3893, 3957, 4041, 4078, 4115, 4131) — none of the six. +Q5_0, IQ4_NL (sgemm.cpp:3829, 3893, 3957, 4041, 4078, 4115, 4131) - none of the six. --- ## 2. Grid and sign tables (shared by both trees) - `iq2xxs_grid` `uint64_t[256]` (ggml-common.h:560), `iq2xs_grid` `uint64_t[512]` (:627), - `iq2s_grid` `uint64_t[1024]` (:758) — 8 weights per entry. -- `iq3xxs_grid` `uint32_t[256]` (:1017), `iq3s_grid` `uint32_t[512]` (:1052) — 4 weights per + `iq2s_grid` `uint64_t[1024]` (:758) - 8 weights per entry. +- `iq3xxs_grid` `uint32_t[256]` (:1017), `iq3s_grid` `uint32_t[512]` (:1052) - 4 weights per entry. - **Every grid byte is a small non-negative magnitude**: iq2xxs bytes are `{0x08,0x19,0x2b}` (ggml-common.h:561-566), iq3xxs `0x04..0x3e` (:1018), iq3s `0x01..0x0f` (:1053). Two things follow: (i) the magnitudes are legal `u8` operands for `VPMADDUBSW`/`VPDPBUSD`, and - the worst-case pair sum `2 × 62 × 127 = 15748` cannot saturate `i16`; (ii) no magnitude is + the worst-case pair sum `2 x 62 x 127 = 15748` cannot saturate `i16`; (ii) no magnitude is zero, so per-byte two's-complement negation inside a 32-bit word never carries across a - byte lane — a fact both trees rely on (ours explicitly, gemm_gen.das:1477-1479). -- `ksigns_iq2xs` `uint8_t[128]` (ggml-common.h:513): 7 sign bits → the 8-bit sign byte with + byte lane - a fact both trees rely on (ours explicitly, gemm_gen.das:1477-1479). +- `ksigns_iq2xs` `uint8_t[128]` (ggml-common.h:513): 7 sign bits -> the 8-bit sign byte with bit 7 set to the parity completion. `kmask_iq2xs[8] = {1,2,4,...,128}` (:509). -- `keven_signs_q2xs` `int8_t[1024]` = 128 × 8 `±1` bytes, defined *per arch file* +- `keven_signs_q2xs` `int8_t[1024]` = 128 x 8 `+/-1` bytes, defined *per arch file* (x86:2624, arm:3595, loongarch:1577, powerpc:1388). Read as `const uint64_t * signs64` - (x86:2675, arm:3646) so one 64-bit load gives 8 `±1` bytes. + (x86:2675, arm:3646) so one 64-bit load gives 8 `+/-1` bytes. --- @@ -116,17 +116,17 @@ sign codes and a 4-bit scale in the top nibble. ### (a) llama.cpp x86 (AVX2), `ggml_vec_dot_iq2_xxs_q8_K`, quants.c:2660-2714 1. Staging: `memcpy(aux32, q2, 16)` into a 16-byte scalar array, aliased as `aux8` - (x86:2677-2678, 2690). Scalar staging of *packed bytes only* — never of decoded weights. -2. Grid compose: `_mm256_set_epi64x(iq2xxs_grid[aux8[3]], ..., aux8[0])` (x86:2691-2692) — + (x86:2677-2678, 2690). Scalar staging of *packed bytes only* - never of decoded weights. +2. Grid compose: `_mm256_set_epi64x(iq2xxs_grid[aux8[3]], ..., aux8[0])` (x86:2691-2692) - four 64-bit table reads written straight into a ymm. No gather instruction, no store. -3. Signs: `signs64[(aux32[1] >> 21) & 127]` etc. (x86:2693-2696) — four 64-bit reads of - `keven_signs_q2xs` composed into a second ymm of `±1` bytes. The parity-completed 8th +3. Signs: `signs64[(aux32[1] >> 21) & 127]` etc. (x86:2693-2696) - four 64-bit reads of + `keven_signs_q2xs` composed into a second ymm of `+/-1` bytes. The parity-completed 8th sign is *in the table*; no parity arithmetic at runtime. -4. Sign application: `_mm256_sign_epi8(q8_1, s2_1)` (x86:2697-2698) — **applied to the +4. Sign application: `_mm256_sign_epi8(q8_1, s2_1)` (x86:2697-2698) - **applied to the activation**, one instruction per 32 bytes. Grid magnitudes stay unsigned. -5. Dot: `_mm256_maddubs_epi16(q2_1, q8s_1)` (x86:2699-2700), u8 × s8 → i16 pairs. -6. Scale: `_mm256_madd_epi16(dot, _mm256_set1_epi16(2*ls+1))` (x86:2703-2704) — the per-32 - scale is folded into the *same* `vpmaddwd` that does the i16→i32 pair reduction. One +5. Dot: `_mm256_maddubs_epi16(q2_1, q8s_1)` (x86:2699-2700), u8 x s8 -> i16 pairs. +6. Scale: `_mm256_madd_epi16(dot, _mm256_set1_epi16(2*ls+1))` (x86:2703-2704) - the per-32 + scale is folded into the *same* `vpmaddwd` that does the i16->i32 pair reduction. One instruction does scale and widening together. 7. Accumulate: two independent i32 ymm chains `sumi1`/`sumi2` (x86:2705-2706), added once per superblock, `cvtepi32_ps` + `fmadd` with `d = x.d * y.d` (x86:2709). The `0.125f` @@ -140,23 +140,23 @@ An SSE/AVX-only fallback exists (x86:2715-2770) using `_mm_set_epi64x` pairs. 1. Same 16-byte `aux32` staging (arm:3644-3645, 3663). 2. Grid compose: `vcombine_s8(vld1_s8(iq2xxs_grid + aux8[0]), vld1_s8(... + aux8[1]))` - (arm:3664-3667) — two 8-byte table loads combined into a q-register. NEON's equivalent of + (arm:3664-3667) - two 8-byte table loads combined into a q-register. NEON's equivalent of `set_epi64x`. 3. Signs: same shape from `signs64` (arm:3668-3671). -4. Sign application: `vmulq_s8(q2u.val[i], q2s.val[i])` (arm:3672-3675) — **applied to the - weights**, because `sdot` is signed×signed and there is no `u8×s8` NEON dot. This is the +4. Sign application: `vmulq_s8(q2u.val[i], q2s.val[i])` (arm:3672-3675) - **applied to the + weights**, because `sdot` is signedxsigned and there is no `u8xs8` NEON dot. This is the arch-forced mirror of the x86 choice. 5. Dot: `ggml_vdotq_s32` chained twice per 32 weights (arm:3676-3677). 6. Scale + accumulate: `sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[1] >> 28))` - (arm:3678-3679) — a **cross-lane reduction and a float multiply-add per 32-weight + (arm:3678-3679) - a **cross-lane reduction and a float multiply-add per 32-weight block**, 8 per superblock. This is the weakest part of llama.cpp's ARM grid kernels. 7. `*s = 0.25f * sumf` (arm:3684). ### (c) What ours does today -Seat: `dasllama_math_gen.das` — the `iq2xxsq8_tile_gen` block, four perms only +Seat: `dasllama_math_gen.das` - the `iq2xxsq8_tile_gen` block, four perms only (`mr=8` NEON sdot; `maddubs/256/8`; `vpdpbusd/256/8`; `vpdpbusd/512/16`), same list as every -other grid format. GEMV generator: `iq2xxs_gemv_gen` → `kq_gemv_gen_impl(gc, 25)` +other grid format. GEMV generator: `iq2xxs_gemv_gen` -> `kq_gemv_gen_impl(gc, 25)` (gemm_gen.das:3344). - Panel alloca `256 * mr` bytes, `align 32`, one per GEMV call @@ -164,7 +164,7 @@ other grid format. GEMV generator: `iq2xxs_gemv_gen` → `kq_gemv_gen_impl(gc, 2 - Per superblock, `emit_iq2xxs_gather` (gemm_gen.das:1749-1806) runs an **`mr`-iteration row loop**; inside, for each of 8 blocks and 4 sub-lanes: - the 4-byte aux word is read as **four separate `i8` loads plus three shifts and three - ORs** (gemm_gen.das:1766-1770) — note IQ3_XXS reads the identical field with **one + ORs** (gemm_gen.das:1766-1770) - note IQ3_XXS reads the identical field with **one aligned `i32` load** (gemm_gen.das:1438-1439). The offsets `(2*blk+1)*mr*4 + 0..3` are contiguous and 4-aligned, so the four-load spelling is pure waste. - `ksigns` lookup: one `i32` load from the `[128 x i32]` global (gemm_gen.das:1633-1687, @@ -172,12 +172,12 @@ other grid format. GEMV generator: `iq2xxs_gemv_gen` → `kq_gemv_gen_impl(gc, 2 - grid: two `i32` loads from the `[1024 x i32]` low/high-word-pair global (gemm_gen.das:1782-1784). - sign apply: `w = (gw ^ mw) + (mw & 0x01010101)` where `mw` is an `i32` from a - 16-entry nibble→byte-mask table (gemm_gen.das:1785-1787) — **three GPR ops and two loads + 16-entry nibble->byte-mask table (gemm_gen.das:1785-1787) - **three GPR ops and two loads per 4 weights**, i.e. ~40 scalar ops per 32 bytes of weight. - store: one 4-byte store into the panel (gemm_gen.das:1793-1794). - The dot then reloads the panel as `<32 x i8>` vectors (gemm_gen.das:1880-1883), takes `abs_w` to recover the magnitudes it just threw away (gemm_gen.das:1888-1889, - 370-375 — `llvm.abs` → `VPABSB`), signs the *splatted activation* with `VPSIGNB` + 370-375 - `llvm.abs` -> `VPABSB`), signs the *splatted activation* with `VPSIGNB` (gemm_gen.das:349-357), then `maddubs` + `pmaddwd` (gemm_gen.das:361-365). - Scale: per-32 `i8` scale row loaded as `<8 x i8>`, `sext` to `<8 x i32>`, then a **`vpmulld` per block per row-lane** (gemm_gen.das:1922-1924, 1935). Integer accumulation across the @@ -190,24 +190,24 @@ are `mr*4 = 32` bytes apart, and the reload at `(blk*16 + j*4)*mr` (gemm_gen.das:1875) is a 32-byte load covering **8 distinct 4-byte stores from 8 different loop iterations**. A wide load spanning several narrow stores is the one direction x86 store-to-load forwarding cannot take. llama.cpp does round-trip through -memory — but only for *indices*, and only in the safe direction (§6c): one wide store, then +memory - but only for *indices*, and only in the safe direction (sec.6c): one wide store, then narrow fully-contained loads. Rough uop budget per 32-byte weight vector (mr = 8, AVX2): -ours ≈ 8×(2 loads + 3 ALU + 1 store) + 1 reload + `abs` + `psign` ≈ 51; -a register-composed form ≈ 9 (`vpinsrd` with folded grid load) + 9 (mask) + `vpor` + -`vpsignb` ≈ 20. +ours ~ 8x(2 loads + 3 ALU + 1 store) + 1 reload + `abs` + `psign` ~ 51; +a register-composed form ~ 9 (`vpinsrd` with folded grid load) + 9 (mask) + `vpor` + +`vpsignb` ~ 20. ### (d) Deltas, ranked | # | Delta | Candidate seat | Why / estimated payoff | |---|---|---|---| | 1 | **Delete the panel.** Compose the 8 grid dwords of one weight vector directly with `insertelement` (the `set_epi32` shape) instead of storing them and reloading. | `tune_perm(dot="maddubs", width=256, mr=8, gather="reg", requires="avx2")` and the `vpdpbusd` twins | Removes 512 scattered 4-byte stores + 64 wide reloads per superblock per row-group, and the store-forwarding hazard. Biggest single item; ~2.5x fewer decode uops. | -| 2 | **Sign in vectors, not GPRs.** Compose a second vector of the 0x00/0xFF byte masks and apply `vpor(mask,1)` + `vpsignb` to the activation, keeping the raw grid vector as the unsigned `maddubs` operand. Deletes both the 3-GPR-op-per-dword negate *and* `llvm.abs`. | folded into `gather="reg"`, or split as `psign="mask"` | ~24 GPR ops → 2 vector ops per 32 bytes; also removes `VPABSB` from the inner loop. | -| 3 | **One `i32` load for the aux word.** Replace the four `i8` loads + 3 shifts + 3 ORs at gemm_gen.das:1766-1770 with the single aligned `i32` load IQ3_XXS already uses (gemm_gen.das:1438-1439). | no seat needed — a straight bug-shaped fix | 6 uops × 8 blocks × mr rows per superblock, for free. Do this regardless. | -| 4 | **Bake the parity-completed sign byte at repack time.** The aux word is 4×7 sign bits + a 4-bit scale, and the scale already moved to the scale row (gemm_gen.das:1746-1748) — so 4 full sign *bytes* fit in the same 4 bytes. Deletes the `ksigns` load entirely and makes the sign bits plane-resident like IQ3_S's. | repack-side; pairs with delta 2 | Removes one dependent table load per 8 weights. Zero size change for this format. | +| 2 | **Sign in vectors, not GPRs.** Compose a second vector of the 0x00/0xFF byte masks and apply `vpor(mask,1)` + `vpsignb` to the activation, keeping the raw grid vector as the unsigned `maddubs` operand. Deletes both the 3-GPR-op-per-dword negate *and* `llvm.abs`. | folded into `gather="reg"`, or split as `psign="mask"` | ~24 GPR ops -> 2 vector ops per 32 bytes; also removes `VPABSB` from the inner loop. | +| 3 | **One `i32` load for the aux word.** Replace the four `i8` loads + 3 shifts + 3 ORs at gemm_gen.das:1766-1770 with the single aligned `i32` load IQ3_XXS already uses (gemm_gen.das:1438-1439). | no seat needed - a straight bug-shaped fix | 6 uops x 8 blocks x mr rows per superblock, for free. Do this regardless. | +| 4 | **Bake the parity-completed sign byte at repack time.** The aux word is 4x7 sign bits + a 4-bit scale, and the scale already moved to the scale row (gemm_gen.das:1746-1748) - so 4 full sign *bytes* fit in the same 4 bytes. Deletes the `ksigns` load entirely and makes the sign bits plane-resident like IQ3_S's. | repack-side; pairs with delta 2 | Removes one dependent table load per 8 weights. Zero size change for this format. | | 5 | **Seat `vpdpbssd`.** The panel bytes are already signed and the activation is signed; `dot_lane` takes that path with no `abs` and no `psign` (gemm_gen.das:344-347, 370-372). Currently unseated for every grid format. | `tune_perm(dot="vpdpbssd", width=256, mr=8, requires="avxvnniint8")` | One-line seat, zero generator change. Only helps AVX-VNNI-INT8 hosts (not zen2/zen4). | -| 6 | **Seat `bias=128`.** Under `gather="reg"` the biased byte is free — `((mag^mask) + (mask&0x01010101)) ^ 0x80808080` — and `bias128` drops `abs` and `psign` on VNNI hosts. Blocked today: `setup_tile_emit` forces `te.bias = 0` whenever `te.kq != 0` (gemm_gen.das:2136). | `tune_perm(dot="vpdpbusd", width=256, mr=8, bias=128, gather="reg", requires="avxvnni\|avx512vnni")` | Needs the `te.bias` clamp lifted for panel formats. Not valid under `maddubs` — `(w+128)×x` pair sums saturate `i16`. | +| 6 | **Seat `bias=128`.** Under `gather="reg"` the biased byte is free - `((mag^mask) + (mask&0x01010101)) ^ 0x80808080` - and `bias128` drops `abs` and `psign` on VNNI hosts. Blocked today: `setup_tile_emit` forces `te.bias = 0` whenever `te.kq != 0` (gemm_gen.das:2136). | `tune_perm(dot="vpdpbusd", width=256, mr=8, bias=128, gather="reg", requires="avxvnni\|avx512vnni")` | Needs the `te.bias` clamp lifted for panel formats. Not valid under `maddubs` - `(w+128)xx` pair sums saturate `i16`. | | 7 | **Fold the scale into `pmaddwd`.** llama.cpp gets scale + widening in one `vpmaddwd` (x86:2703). We cannot broadcast a scalar scale (each lane is a different row), but we can keep an `<16 x i16>` per-row scale vector and use `pmaddwd` against the `maddubs` output instead of `vpmulld` on the i32 accumulator. | `scale="madd16"` | Replaces a 10-cycle `vpmulld` per block per row-lane with a 5-cycle `vpmaddwd` that also removes the separate ones-vector `pmaddwd`. Modest but cheap. | --- @@ -233,7 +233,7 @@ sign is parity. Per-16 scales in a nibble-packed `scales[QK_K/32]`. 4. **Sign expansion by shuffle + cmpeq.** For each 32-byte group: `vpshufb` broadcasts the group's sign byte across 8 lanes (`block_sign_shuffle_1/2`, x86:2796-2803), `AND` with `bit_selector_mask` = `{1,2,4,...,128}` repeated (x86:2804-2807), `cmpeq` against the same - mask → 0x00/0xFF, `OR` with `mone` → `±1`, then `_mm256_sign_epi8(q8, ...)` + mask -> 0x00/0xFF, `OR` with `mone` -> `+/-1`, then `_mm256_sign_epi8(q8, ...)` (x86:2871-2885). **Signs on the activation; grid stays unsigned.** Four vector ops per 32 weights. 5. Scales: one 8-byte `memcpy`, nibble split and interleave, `scales = (stmp<<1) | 1` @@ -245,9 +245,9 @@ An SSE path mirrors all of this at 128 bits (x86:2909-3070). ### (b) llama.cpp arm64, quants.c:3693-3763 -1. **No parity arithmetic** — `signs64 + (q2[i] >> 9)` reads `keven_signs_q2xs` directly with +1. **No parity arithmetic** - `signs64 + (q2[i] >> 9)` reads `keven_signs_q2xs` directly with the 7-bit code (arm:3739-3742), and the table already carries the parity completion. The - x86 `pshufb` parity trick exists only because a 128-entry × 8-byte table read does not + x86 `pshufb` parity trick exists only because a 128-entry x 8-byte table read does not vectorize on x86 the way `vld1_s8` does on NEON. 2. Grid: `vcombine_s8(vld1_s8(iq2xs_grid + (q2[0] & 511)), ...)` (arm:3735-3738). 3. Signs on the **weights**, `vmulq_s8` (arm:3743-3746). @@ -265,16 +265,16 @@ An SSE path mirrors all of this at 128 bits (x86:2909-3070). `emit_iq2xs_gather` (gemm_gen.das:1689-1745). Identical skeleton to IQ2_XXS: - The `uint16` word is assembled from **two separate `i8` loads plus a shift and an OR** - (gemm_gen.das:1711-1716). The offsets are `lp%4 ∈ {0,2}` within the same 4-byte column, so + (gemm_gen.das:1711-1716). The offsets are `lp%4 in {0,2}` within the same 4-byte column, so a single 2-byte load would do. -- `ksigns` lookup from the 7-bit high field (gemm_gen.das:1718-1720) — the parity path, same +- `ksigns` lookup from the 7-bit high field (gemm_gen.das:1718-1720) - the parity path, same as NEON's, not x86's `pshufb` trick. Correct choice for a scalar decode. - Grid: two `i32` loads from the `[1024 x i32]` low/high pair global (gemm_gen.das:1578-1632, 1725-1727). - Same 3-GPR-op negate and same 4-byte panel store (gemm_gen.das:1728-1732). - Consumption: `iq2 = true` (gemm_gen.das:1817), so the lo and hi 16-weight halves keep **split accumulators** `a[]`/`a1[]` and get separate per-16 unsigned scales - (gemm_gen.das:1893-1898, 1912-1919, 1932-1937) — the per-16 granularity IQ2_XS and IQ2_S + (gemm_gen.das:1893-1898, 1912-1919, 1932-1937) - the per-16 granularity IQ2_XS and IQ2_S need. That part is right and is structurally what ARM's `vpaddq` + `vmlaq_s32` achieves, without the reduction. @@ -284,7 +284,7 @@ Same 1/2/5/6/7 as IQ2_XXS, plus: | # | Delta | Candidate seat | Why | |---|---|---|---| -| 3' | **One `i16` load for the packed word.** Replace the two `i8` loads + shift + OR (gemm_gen.das:1711-1716) with one aligned 2-byte load. | none — direct fix | 3 uops × 32 × mr per superblock, free. | +| 3' | **One `i16` load for the packed word.** Replace the two `i8` loads + shift + OR (gemm_gen.das:1711-1716) with one aligned 2-byte load. | none - direct fix | 3 uops x 32 x mr per superblock, free. | | 4' | **Split the sign byte into the plane at repack time.** IQ2_XS is the one format where the parity-completed byte does not fit for free (9 + 8 = 17 bits > 16). Costs +1 bit per 8 weights if adopted; race it against the `ksigns` load. | repack-side | Lower priority than for IQ2_XXS / IQ3_XXS where it is free. | | 8 | **Race the `pshufb` parity form** as an alternative to the `ksigns` table read, once the sign path is vectorized (delta 2). llama.cpp measured it as *"a significant boost"* (x86:2822) on their layout. | `psign="parity"` | Only interesting if delta 2 lands first; on a scalar-index decode the table read wins. | @@ -299,14 +299,14 @@ per-16 scales. ### (a) llama.cpp x86 (AVX2), quants.c:3075-3160 1. Grid compose: `_mm256_set_epi64x(iq2s_grid[qs[3] | ((qh[ib32] << 2) & 0x300)], ...)` - (x86:3123-3130) — index arithmetic done **scalar, in the address expression**, four + (x86:3123-3130) - index arithmetic done **scalar, in the address expression**, four 64-bit reads per ymm. Note the shift amounts run `8,6,4,2` down the lanes so the same `qh` byte feeds all four. 2. Sign expansion: `_mm256_set1_epi32(signs[0] | (signs[1] << 16))` broadcasts 4 sign bytes, `shuffle_epi8(aux256, mask1)` spreads each byte across 8 lanes (`k_mask1`, x86:3090-3092), - `AND` with `k_mask2` = `{1,2,...,128}`×4 (x86:3094-3096), `cmpeq` → 0x00/0xFF + `AND` with `k_mask2` = `{1,2,...,128}`x4 (x86:3094-3096), `cmpeq` -> 0x00/0xFF (x86:3133-3135). -3. Sign application: `_mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1)` (x86:3136) — a +3. Sign application: `_mm256_sub_epi8(_mm256_xor_si256(s2_1, q8_1), s2_1)` (x86:3136) - a **two-instruction conditional negate under a 0/-1 mask**, cheaper than `or(mask,1)` + `psign` used by IQ2_XS, and applied **to the activation**. 4. `maddubs` (x86:3146-3147), then scale via @@ -320,12 +320,12 @@ per-16 scales. 1. Grid: four `vcombine_s8(vld1_s8(iq2s_grid + (qs[i] | ((qh[..] << k) & 0x300))), ...)` (arm:3808-3816). 2. Sign expansion: `vdupq_n_u32(signs[0] | (signs[1]<<16))`, two `ggml_vqtbl1q_u8` with - `k_mask1` halves, `vandq_u8` with `k_mask2`, `vceqq_u8` (arm:3820-3825) — the direct + `k_mask1` halves, `vandq_u8` with `k_mask2`, `vceqq_u8` (arm:3820-3825) - the direct NEON analogue of the x86 shuffle+cmpeq. -3. Sign application: `vmulq_s8(vorrq_u8(vs, m1), q2s)` (arm:3826-3827) — `±1` on the +3. Sign application: `vmulq_s8(vorrq_u8(vs, m1), q2s)` (arm:3826-3827) - `+/-1` on the **weights**. 4. Four `ggml_vdotq_s32` (arm:3838-3841). -5. Scale: `sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32] & 0xf))` (arm:3843-3846) — +5. Scale: `sumi1 += vaddvq_s32(p1) * (1 + 2*(x[i].scales[ib32] & 0xf))` (arm:3843-3846) - scalar integer accumulate, **four `vaddvq_s32` cross-lane reductions per 64 weights**. Weaker than the IQ2_XS `vpaddq`+`vmlaq_s32` form on the same arch, for no obvious reason. @@ -336,11 +336,11 @@ per-16 scales. - Index: one `i8` load of `qs`, one `i8` load of the `qh` byte per block, `shl`/`and`/`or` to build the 10-bit index, then `shl 1` for the low/high word pair (gemm_gen.das:1543-1550). -- **Sign byte is read straight out of the plane** (gemm_gen.das:1541-1543) — no `ksigns` +- **Sign byte is read straight out of the plane** (gemm_gen.das:1541-1543) - no `ksigns` indirection, because IQ2_S stores signs explicitly. This is the format where delta 2 (vector sign expansion) is cheapest to reach: the sign bits are already plane-resident and, given the `(p/4)*mr*4 + p%4` column layout (gemm_gen.das:1542), **one 32-byte load - covers 8 rows × 4 consecutive sign bytes at stride 4** — exactly the input a + covers 8 rows x 4 consecutive sign bytes at stride 4** - exactly the input a `vpshufb`+`cmpeq` expansion wants. - Same 3-GPR-op negate, same 4-byte panel store (gemm_gen.das:1557-1562). - Consumption: the `iq2` split-accumulator per-16 path (gemm_gen.das:1817, 1893-1898). @@ -349,10 +349,10 @@ per-16 scales. | # | Delta | Candidate seat | Why | |---|---|---|---| -| 1 | Delete the panel (as §3). | `gather="reg"` | Same as every format. | +| 1 | Delete the panel (as sec.3). | `gather="reg"` | Same as every format. | | 2* | **Vector sign expansion, promoted.** For IQ2_S specifically the 8 rows' sign bytes are one 32-byte strided column load away (gemm_gen.das:1542); `vpshufb` to select the wanted byte per row, `vpshufb`/`and`/`cmpeq` to expand to 32 mask bytes. ~5 vector ops replace 8 loads + 8 GPR negates. | `psign="expand"` | Highest-value sign delta of the five, because no `ksigns` table read stands in the way. | -| 3 | Adopt the **`xor`+`sub` conditional negate** (x86:3136) in place of `or(mask,1)`+`psign` wherever the mask is already 0/-1. | folded into `psign="expand"` | 2 ops vs 2 ops on x86, but frees `VPSIGNB` port pressure and works at 512 bits where `llvm.x86.avx2.psign.b` does not exist (see §9). | -| 5,6,7 | `vpdpbssd` seat, `bias=128` seat, `pmaddwd` scale fold — as §3. | | | +| 3 | Adopt the **`xor`+`sub` conditional negate** (x86:3136) in place of `or(mask,1)`+`psign` wherever the mask is already 0/-1. | folded into `psign="expand"` | 2 ops vs 2 ops on x86, but frees `VPSIGNB` port pressure and works at 512 bits where `llvm.x86.avx2.psign.b` does not exist (see sec.9). | +| 5,6,7 | `vpdpbssd` seat, `bias=128` seat, `pmaddwd` scale fold - as sec.3. | | | --- @@ -364,7 +364,7 @@ four 7-bit sign codes + 4-bit scale, at `qs + QK_K/4`. ### (a) llama.cpp x86 (AVX2), quants.c:3260-3316 1. Grid compose: `_mm256_set_epi32(iq3xxs_grid[q3[7]], ..., iq3xxs_grid[q3[0]])` - (x86:3290-3295) — **eight 32-bit table reads written directly into one ymm**, twice per + (x86:3290-3295) - **eight 32-bit table reads written directly into one ymm**, twice per loop iteration. This is the exact composition our layout also needs. 2. `memcpy(aux32, gas, 8)` staging (x86:3296). 3. Signs: four `signs64[(aux32[k] >> 7j) & 127]` per ymm (x86:3297-3300), `keven` table, @@ -375,40 +375,40 @@ four 7-bit sign codes + 4-bit scale, at `qs + QK_K/4`. ### (b) llama.cpp arm64, quants.c:3864-3922 -1. Grid compose: `ggml_vld1q_u32(iq3xxs_grid[q3[0]], ..., q3[3])` (arm:3896-3899) — the macro +1. Grid compose: `ggml_vld1q_u32(iq3xxs_grid[q3[0]], ..., q3[3])` (arm:3896-3899) - the macro (ggml-cpu-impl.h:82-84) is a brace-initialised `uint32x4_t`, i.e. four scalar dword reads the compiler materialises with `ld1 {v.s}[lane]` or `ins`. Same shape as `set_epi32`. 2. Signs from `keven_signs_q2xs`, `vcombine_s8` of two 8-byte reads (arm:3901-3904). 3. `vmulq_s8` on the **weights** (arm:3905-3908). -4. Chained `ggml_vdotq_s32` ×2 per 32 weights (arm:3909-3910). -5. `sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[0] >> 28))` (arm:3911-3912) — again the +4. Chained `ggml_vdotq_s32` x2 per 32 weights (arm:3909-3910). +5. `sumf1 += vaddvq_s32(p1) * (0.5f + (aux32[0] >> 28))` (arm:3911-3912) - again the per-block cross-lane reduction and **float** accumulate; `*s = 0.5f * sumf` (arm:3916). ### (c) What ours does today `emit_iq3xxs_gather` (gemm_gen.das:1418-1474). Cleanest of our five gathers: -- **One aligned `i32` load** for the aux word (gemm_gen.das:1438-1439) — the spelling +- **One aligned `i32` load** for the aux word (gemm_gen.das:1438-1439) - the spelling IQ2_XXS should copy. - `ksigns` lookup per sub-lane (gemm_gen.das:1442-1444). - Two `i8` index loads (gemm_gen.das:1445-1450); grid is the **halved** `[256 x i32]` global (gemm_gen.das:1362-1377, 1455-1457). - Same 3-GPR-op negate + 4-byte panel store (gemm_gen.das:1458-1462). - Consumption: `iq2 = false` (gemm_gen.das:1817), so lo and hi share one accumulator and one - per-32 signed scale (gemm_gen.das:1899-1901, 1920-1921) — matching the per-32 granularity. + per-32 signed scale (gemm_gen.das:1899-1901, 1920-1921) - matching the per-32 granularity. ### (d) Deltas, ranked -1. **Delete the panel** (`gather="reg"`) — the single biggest item, and the composition it +1. **Delete the panel** (`gather="reg"`) - the single biggest item, and the composition it needs is literally x86:3290's `set_epi32`, transposed from "8 grid words of one row" to "one grid word from each of 8 rows". -2. **Vector sign apply** (`psign="mask"` / folded into delta 1) — deletes 24 GPR ops and +2. **Vector sign apply** (`psign="mask"` / folded into delta 1) - deletes 24 GPR ops and `VPABSB` per 32-byte vector. -3. **Bake the parity-completed sign byte at repack time.** Aux is 4×7 sign bits + a 4-bit - scale, and the scale already lives on the scale row — so four full sign bytes fit in the +3. **Bake the parity-completed sign byte at repack time.** Aux is 4x7 sign bits + a 4-bit + scale, and the scale already lives on the scale row - so four full sign bytes fit in the same 4 bytes, exactly as for IQ2_XXS. Free; deletes the `ksigns` load. 4. `vpdpbssd` seat; `bias=128` seat (needs the `te.bias` clamp lifted, gemm_gen.das:2136); - `pmaddwd` scale fold — as §3. + `pmaddwd` scale fold - as sec.3. --- @@ -426,33 +426,33 @@ sign bytes** (`x[i].signs`); nibble-packed per-32 scales. 2. **The index goes through memory, in the safe direction.** `index_t` is a union of `__m256i vec[2]` and `uint32_t index[16]` (x86:3406-3411); the vector is stored and the dwords are read back scalar to address the grid (x86:3441-3450). One wide store feeding - 16 narrow, fully-contained loads forwards fine — the opposite of our panel's one wide + 16 narrow, fully-contained loads forwards fine - the opposite of our panel's one wide load over 8 narrow stores. 3. **An explicit negative result on hardware gather**, x86:3440: `// At leat on my CPU (Ryzen 7950X), using _mm256_i32gather_epi32 is slower than - _mm256_set_epi32. Strange.` — with the `vpgatherdd` version left commented out at + _mm256_set_epi32. Strange.` - with the `vpgatherdd` version left commented out at x86:3441-3442. Do not spend time on a gather-instruction spelling. -4. Sign expansion: identical to IQ2_S — `set1_epi32(signs[0] | signs[1]<<16)`, +4. Sign expansion: identical to IQ2_S - `set1_epi32(signs[0] | signs[1]<<16)`, `shuffle_epi8(mask1)`, `and(mask2)`, `cmpeq` (x86:3452-3459). -5. Sign application: `sub(xor(s2, q8), s2)` (x86:3455, 3460) — **on the activation**. +5. Sign application: `sub(xor(s2, q8), s2)` (x86:3455, 3460) - **on the activation**. 6. `maddubs` (x86:3464-3465); scale `2*ls+1` from the nibble pair via `set1_epi16` + - `madd_epi16` (x86:3466-3469); two i32 accumulators; `fmadd`; **no global factor** — + `madd_epi16` (x86:3466-3469); two i32 accumulators; `fmadd`; **no global factor** - `*s = hsum_float_8(accumf)` (x86:3475). ### (b) llama.cpp arm64, quants.c:3926-4032 1. Index compose in vector: `vorrq_u16(vmovl_u8(qs_half), vandq_u16(vshlq_u16(vdupq_n_u16(qh), hshift), m256))` with `k_shift = {8,7,...,1}` (arm:3985-3986, 3992). -2. Same union round trip — `vec_index_t { uint16x8_t vec_index; uint16_t index[8]; }` +2. Same union round trip - `vec_index_t { uint16x8_t vec_index; uint16_t index[8]; }` (arm:3944-3947), stored then read back scalar for `ggml_vld1q_u32(iq3s_grid[idx.index[0]], ...)` (arm:3987-3996). -3. Sign expansion via `vqtbl1q_u8` + `vceqq_u8` + `vorrq_u8(..., m1)` → `±1` +3. Sign expansion via `vqtbl1q_u8` + `vceqq_u8` + `vorrq_u8(..., m1)` -> `+/-1` (arm:3998-4012); applied to the **weights** with `vmulq_s8` (arm:4004-4005, 4016-4017). 4. **Scales precomputed with a SWAR trick**: `memcpy(scales32, x[i].scales, 4)`, then `scales32[1] = (((scales32[0] >> 4) & 0x0f0f0f0f) << 1) | 0x01010101` and the twin for the - low nibbles (arm:3976-3979) — eight `(2s+1)` scale bytes computed with four GPR ops per + low nibbles (arm:3976-3979) - eight `(2s+1)` scale bytes computed with four GPR ops per superblock. -5. `sumi1 += vaddvq_s32(p1) * scales8[ib32/2]` (arm:4020-4023) — per-block cross-lane +5. `sumi1 += vaddvq_s32(p1) * scales8[ib32/2]` (arm:4020-4023) - per-block cross-lane reduction, scalar integer accumulate; `*s = sumf` (arm:4028). ### (c) What ours does today @@ -461,11 +461,11 @@ sign bytes** (`x[i].signs`); nibble-packed per-32 scales. - `qh` byte load per block (gemm_gen.das:1319-1321), sign byte load per sub-lane (gemm_gen.das:1323-1325), `qs` byte load + `shl`/`and`/`or` per grid word - (gemm_gen.das:1327-1333). **All scalar, one byte at a time** — no `sllv`-style vector index + (gemm_gen.das:1327-1333). **All scalar, one byte at a time** - no `sllv`-style vector index compose. - Grid: `[512 x i32]` private constant, `align 64` (gemm_gen.das:1260-1272), one `i32` load per 4 weights (gemm_gen.das:1339-1341). -- Sign: 16-entry nibble→byte-mask table (`dasllama.iq3s.smask`, gemm_gen.das:1274-1293), then +- Sign: 16-entry nibble->byte-mask table (`dasllama.iq3s.smask`, gemm_gen.das:1274-1293), then `(gw ^ mw) + (mw & 0x01010101)` (gemm_gen.das:1342-1344). - 4-byte panel store (gemm_gen.das:1345-1347). - Consumption: shared per-32 signed scale path (gemm_gen.das:1899-1901, 1920-1921); scale @@ -476,12 +476,12 @@ sign bytes** (`x[i].signs`); nibble-packed per-32 scales. | # | Delta | Candidate seat | Why | |---|---|---|---| -| 1 | **Delete the panel** — compose 8 grid dwords per weight vector with `insertelement`. | `tune_perm(dot="maddubs", width=256, mr=8, gather="reg", requires="avx2")` | The 0.92x tail's stated cause, and the measurement in followup 61 supports it. | -| 2 | **Vector sign expansion off the plane.** Like IQ2_S, the sign bytes are explicit and plane-resident at stride 4 across rows (gemm_gen.das:1324) — one 32-byte column load + `vpshufb`/`and`/`cmpeq` gives the full 32-byte `±1`/mask vector. | `psign="expand"` | Deletes the `smask` table read *and* the 3-GPR-op negate *and* `llvm.abs`. | -| 3 | **Vectorized index compose** (`sllv` shape, x86:3430-3438 / arm:3985-3986) feeding a *wide-store / narrow-load* union — the safe memory direction — instead of per-byte `qs` loads. | `idx="vec"` | Only worth racing after delta 1; the win is ~4 GPR ops per grid word. Note llama.cpp deliberately keeps the *grid read* scalar even here. | -| 4 | Do **not** race `vpgatherdd` for the grid read. | — | x86:3440 records it losing to `set_epi32` on Zen 4. | -| 5 | Adopt the ARM SWAR scale expansion (arm:3976-3979) if the scale row is ever computed at runtime. | — | Ours pre-bakes scales in the repack, so this is already better; noted for completeness. | -| 6,7 | `vpdpbssd` seat; `bias=128` seat; `pmaddwd` scale fold — as §3. | | | +| 1 | **Delete the panel** - compose 8 grid dwords per weight vector with `insertelement`. | `tune_perm(dot="maddubs", width=256, mr=8, gather="reg", requires="avx2")` | The 0.92x tail's stated cause, and the measurement in followup 61 supports it. | +| 2 | **Vector sign expansion off the plane.** Like IQ2_S, the sign bytes are explicit and plane-resident at stride 4 across rows (gemm_gen.das:1324) - one 32-byte column load + `vpshufb`/`and`/`cmpeq` gives the full 32-byte `+/-1`/mask vector. | `psign="expand"` | Deletes the `smask` table read *and* the 3-GPR-op negate *and* `llvm.abs`. | +| 3 | **Vectorized index compose** (`sllv` shape, x86:3430-3438 / arm:3985-3986) feeding a *wide-store / narrow-load* union - the safe memory direction - instead of per-byte `qs` loads. | `idx="vec"` | Only worth racing after delta 1; the win is ~4 GPR ops per grid word. Note llama.cpp deliberately keeps the *grid read* scalar even here. | +| 4 | Do **not** race `vpgatherdd` for the grid read. | - | x86:3440 records it losing to `set_epi32` on Zen 4. | +| 5 | Adopt the ARM SWAR scale expansion (arm:3976-3979) if the scale row is ever computed at runtime. | - | Ours pre-bakes scales in the repack, so this is already better; noted for completeness. | +| 6,7 | `vpdpbssd` seat; `bias=128` seat; `pmaddwd` scale fold - as sec.3. | | | --- @@ -490,7 +490,7 @@ sign bytes** (`x[i].signs`); nibble-packed per-32 scales. Entry 61 says: > The stamped gemv gathers each superblock into an alloca panel and then runs the vector dot -> — a store/load round trip per superblock that a single token never amortizes; llama.cpp's +> - a store/load round trip per superblock that a single token never amortizes; llama.cpp's > per-row form (grid words composed straight into vectors, signs applied to the ACTIVATION > via shuffle+cmpeq/xor-sub, magnitudes kept unsigned for maddubs) edges it 57.0 vs 52.4 > tg128 on the zen2. @@ -516,9 +516,9 @@ Entry 61 says: - *"signs applied to the ACTIVATION"* is **not portable to our layout.** It is true of llama.cpp's x86 kernels only, and it is a consequence of their one-row-at-a-time shape. In our `mr`-interleaved GEMV the lanes of one weight vector are 8 different output rows - with 8 different sign patterns, while the activation dword is splatted across all lanes — + with 8 different sign patterns, while the activation dword is splatted across all lanes - so a single `psign` on the activation cannot be correct. llama.cpp's own **arm64** kernels - make the same concession for the same reason (`sdot` is signed×signed): they apply signs to + make the same concession for the same reason (`sdot` is signedxsigned): they apply signs to the **weights** with `vmulq_s8` at arm:3672, 3743, 3826, 3905, 4004. Adopting "signs-on-activation" would mean abandoning row interleave (`mr = 1`), which would give back the per-row horizontal reduction we currently avoid entirely. Not worth it. @@ -526,33 +526,33 @@ Entry 61 says: unsigned and build the sign mask as a vector**, so `psign(x_splat, mask|1)` replaces `abs(panel_bytes)` + `psign(x_splat, panel_bytes)` and the 3-GPR-op-per-dword negate disappears with the panel. The `abs`+`psign` pair is dropped outright only by - `vpdpbssd` (signed×signed, gemm_gen.das:344-347) or by `bias=128` - (gemm_gen.das:339-343) — neither of which is seated for any grid format today. + `vpdpbssd` (signedxsigned, gemm_gen.das:344-347) or by `bias=128` + (gemm_gen.das:339-343) - neither of which is seated for any grid format today. - Entry 61 is also **under-scoped**: it is filed as an IQ3_S item, but the panel, the gather shape, the scalar sign negate and the `abs`+`psign` pair are byte-for-byte identical in all five gathers. One `gather="reg"` spelling in `emit_block_iq4xs` covers all five at once. --- -## 9. Cross-format section — what one spelling would cover +## 9. Cross-format section - what one spelling would cover All five gathers are the same function with a different index/sign preamble. Ranked by estimated decode-shape payoff: -### Tier 1 — one change, all five formats +### Tier 1 - one change, all five formats **`gather="reg"`: compose the weight vector in registers, delete the panel.** Where: `emit_block_iq4xs` (gemm_gen.das:1811-1957) and the five gather emitters. Restructure the gather loop from *row-major* (`for r in mr { for blk, l, h }`, gemm_gen.das:1305-1352) to *group-major* (`for blk, j, half { for r in mr }`), emitting `insertelement` of each row's -decoded dword into an `` and bitcasting to `te.vwi8` — the same value +decoded dword into an `` and bitcasting to `te.vwi8` - the same value `load_vec(te, te.vwi8, wbase, noff)` returns today (gemm_gen.das:1880-1883), produced without touching memory. Why it is the top item: -- Removes `256 × mr / 4 = 512` scattered 4-byte stores per superblock (mr = 8) and the +- Removes `256 x mr / 4 = 512` scattered 4-byte stores per superblock (mr = 8) and the 64 wide reloads, plus the `alloca` traffic. - Removes the **wrong-direction store-forwarding hazard**: the reload at `(blk*16 + j*4)*mr` (gemm_gen.das:1875) is a 32-byte load spanning 8 distinct 4-byte @@ -562,9 +562,9 @@ Why it is the top item: `vec_index_t`) are all in the *safe* direction and carry indices, never decoded weights. - Worse at width 512 today, better after: the `mr = 16` seat writes 1024 stores into a 4096-byte panel and reloads with 64-byte loads spanning 16 stores each. -- Rough per-32-byte-vector uop budget: ~51 now → ~20 composed. +- Rough per-32-byte-vector uop budget: ~51 now -> ~20 composed. -Seats (one per existing perm row, five formats × four rows): +Seats (one per existing perm row, five formats x four rows): ``` tune_perm(mr = 8, gather = "reg"), @@ -577,13 +577,13 @@ tune_perm(dot = "vpdpbusd", width = 512, mr = 16, gather = "reg", requires = "av `name_value`, so these stamp as `..._gather_reg`; add them to each `tune(... fallback = ...)` chain in priority order.) -### Tier 2 — one change, all five formats +### Tier 2 - one change, all five formats **`psign="mask"` / `psign="expand"`: build the sign vector, stop negating in GPRs.** -Composed with tier 1: keep the raw grid dword as the unsigned magnitude (it already is — +Composed with tier 1: keep the raw grid dword as the unsigned magnitude (it already is - ggml-common.h:561, 1018, 1053), compose a second vector of 0x00/0xFF masks, and apply -`vpor(mask, 1)` + `VPSIGNB` to the splatted activation — or the cheaper +`vpor(mask, 1)` + `VPSIGNB` to the splatted activation - or the cheaper `sub(xor(mask, x), mask)` conditional negate llama.cpp uses at x86:3136/3455. Deletes: - the three GPR ops per dword at gemm_gen.das:1342-1344 / 1458-1460 / 1557-1559 / @@ -592,9 +592,9 @@ ggml-common.h:561, 1018, 1053), compose a second vector of 0x00/0xFF masks, and Two sub-variants worth racing separately: -- `psign="mask"` — compose the mask vector from the existing 16-entry `smask` global with +- `psign="mask"` - compose the mask vector from the existing 16-entry `smask` global with `insertelement` (8 more folded loads). -- `psign="expand"` — read the sign bits vectorized off the plane and expand with +- `psign="expand"` - read the sign bits vectorized off the plane and expand with `vpshufb`+`and`+`cmpeq` (llama.cpp's `k_mask1`/`k_mask2`, x86:3090-3096 / arm:3782-3786). Cheapest for **IQ2_S** and **IQ3_S**, whose sign bytes are plane-resident at stride 4 across rows (gemm_gen.das:1542, 1324); needs one `ksigns` scalar read per 8 weights for @@ -604,10 +604,10 @@ Note `llvm.x86.avx2.psign.b` is only wired at `width == 256` (gemm_gen.das:2185- 512-bit leg already falls back to a `select` (gemm_gen.das:358-362). The `xor`+`sub` form works at both widths and would simplify that split. -### Tier 3 — seats only, zero generator change +### Tier 3 - seats only, zero generator change **`dot = "vpdpbssd"` for all five formats.** The panel/composed bytes are signed and the -activation is signed, so `dot_lane` takes the native `s8·s8` path with no `abs` and no +activation is signed, so `dot_lane` takes the native `s8*s8` path with no `abs` and no `psign` (gemm_gen.das:344-347, 370-372). `perm_declines` already permits it at width 256 under `avxvnniint8` (gemm_gen.das:212-220). One line per format: @@ -618,23 +618,23 @@ tune_perm(dot = "vpdpbssd", width = 256, mr = 8, requires = "avxvnniint8"), Only helps AVX-VNNI-INT8 hosts (not zen2, not zen4), so file it as cheap coverage, not as the fix for the measured tail. -### Tier 4 — repack-side, three of five formats +### Tier 4 - repack-side, three of five formats **Bake the parity-completed sign byte into the plane.** `ksigns_iq2xs` maps a 7-bit code to the 8-bit sign byte (ggml-common.h:513). For **IQ2_XXS** and **IQ3_XXS** the aux word is -4×7 sign bits + a 4-bit scale, and the scale nibble already moved to the scale row -(gemm_gen.das:1746-1748) — so four full sign *bytes* occupy the same four bytes, at zero +4x7 sign bits + a 4-bit scale, and the scale nibble already moved to the scale row +(gemm_gen.das:1746-1748) - so four full sign *bytes* occupy the same four bytes, at zero size cost, and the `ksigns` load disappears (gemm_gen.das:1442-1444, 1775-1776). For **IQ2_XS** it costs +1 bit per 8 weights (9 index bits + 8 sign bits > 16), so race it. This also promotes IQ2_XXS/IQ3_XXS into the cheap `psign="expand"` class alongside IQ2_S/IQ3_S. -### Tier 5 — small, local, do anyway +### Tier 5 - small, local, do anyway - **IQ2_XXS aux word**: four `i8` loads + 3 shifts + 3 ORs (gemm_gen.das:1766-1770) where IQ3_XXS uses one aligned `i32` load of the identically-aligned field - (gemm_gen.das:1438-1439). 6 uops × 8 blocks × mr rows per superblock, free to reclaim. + (gemm_gen.das:1438-1439). 6 uops x 8 blocks x mr rows per superblock, free to reclaim. - **IQ2_XS packed word**: two `i8` loads + shift + OR (gemm_gen.das:1711-1716) where one - 2-byte load suffices (`lp % 4 ∈ {0,2}`, so the pair never straddles the 4-byte column). + 2-byte load suffices (`lp % 4 in {0,2}`, so the pair never straddles the 4-byte column). - **Panel/vector load alignment**: `load_vec` hard-codes `align 1` (gemm_gen.das:376-379) although the alloca is `align 32` (gemm_gen.das:3082) and every offset is a multiple of `mr*4`. Moot once the panel is gone, but it also affects the tile's @@ -645,10 +645,10 @@ also promotes IQ2_XXS/IQ3_XXS into the cheap `psign="expand"` class alongside IQ - **`bias = 128` is clamped off** for every `kq` format at gemm_gen.das:2136 (`te.bias = (te.mx4 || te.q51 || te.kq != 0) ? 0 : p.bias`). Since we synthesize the panel bytes ourselves, `w ^ 0x80808080` is a free per-byte `+128` (no carries), so lifting the - clamp for the five panel formats would make the `bias128` lattice — which drops `abs` and - `psign` entirely and folds the `-128·Σx` correction into the accumulator init - (`bias_acc_init`, gemm_gen.das:400-427) — available on VNNI hosts. **Not** valid under - `maddubs`: `(w+128) × x` pair sums reach 65280 and saturate `i16`, which is why the knob is + clamp for the five panel formats would make the `bias128` lattice - which drops `abs` and + `psign` entirely and folds the `-128*Sumx` correction into the accumulator init + (`bias_acc_init`, gemm_gen.das:400-427) - available on VNNI hosts. **Not** valid under + `maddubs`: `(w+128) x x` pair sums reach 65280 and saturate `i16`, which is why the knob is documented `vpdpbusd`-only (gemm_gen.das:44-47, 170-171). ### What we already do better, and should not regress @@ -660,11 +660,11 @@ also promotes IQ2_XXS/IQ3_XXS into the cheap `psign="expand"` class alongside IQ - **Global factor pre-baked.** llama.cpp multiplies by `0.125f` / `0.25f` / `0.5f` per row at the end (x86:2713, 3161, 3316; arm:3684, 3760, 3916). Ours rides the repacked scale row. - **Per-16 split accumulators for IQ2_S/IQ2_XS** (gemm_gen.das:1817, 1893-1898) match the - formats' actual scale granularity without any extra reduction — structurally what + formats' actual scale granularity without any extra reduction - structurally what llama.cpp's best ARM arm (IQ2_XS `vpaddq`+`vmlaq_s32`, arm:3751-3752) achieves, and better - than what its IQ2_S arm does (scalar `vaddvq` × scalar scale, arm:3843-3846). + than what its IQ2_S arm does (scalar `vaddvq` x scalar scale, arm:3843-3846). - **Q2_K needs no work.** Our `k2` arm decodes 2-bit lanes in-register with a shift and an - `and` (gemm_gen.das:1049-1056) — the same technique as llama.cpp AVX2 (x86:1626-1629) and - NEON (arm:1988-1996) — with no panel, no gather, no table. That is exactly why it measures + `and` (gemm_gen.das:1049-1056) - the same technique as llama.cpp AVX2 (x86:1626-1629) and + NEON (arm:1988-1996) - with no panel, no gather, no table. That is exactly why it measures at parity, and it is the control that proves the gap in the other five is the *decode*, not the dot or the fold. From 882c2e2c4dae14b9923a67a7b317ae68449f9013 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 00:54:04 -0700 Subject: [PATCH 004/123] kq_kernel_bench: =name selects one registry row exactly Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index d7b74b229b..06895e37fe 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -38,7 +38,7 @@ struct BenchArgs { @clarg_doc = "Skip the GEMV (decode) arm" no_gemv : bool @clarg_short = "p" - @clarg_doc = "Registry rows to time - comma-separated substrings of the perm suffix (default: every row, the reference body included)" + @clarg_doc = "Registry rows to time - comma-separated substrings of the perm suffix, =name for an exact row (default: every row, the reference body included)" perm : string @clarg_doc = "TSV rows only, no banner" tsv : bool @@ -270,7 +270,10 @@ def perm_wanted(spec, suffix : string) : bool { } for (tok in split(spec, ",")) { let tk = strip(tok) - if (!empty(tk) && find(suffix, tk) >= 0) { + if (empty(tk)) { + continue + } + if (starts_with(tk, "=") ? suffix == slice(tk, 1) : find(suffix, tk) >= 0) { return true } } From ba586e68ddd86deb17a14d8b4212f34300e4a5a2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:00:12 -0700 Subject: [PATCH 005/123] kernel parity, CPU item 3: the iq2 formats' u64 grid pair is one 8-byte load iq2s, iq2xs and iq2xxs index the u64 grid as an even/odd dword pair; both halves now come from one load with the high half split off in registers (the h = 0 and h = 1 decodes share it through CSE). With sign=vec, one thread, m=4096 k=14336: iq2s 7076 -> 5152 us (reference 5074), iq2xxs 7832 -> 5209 (5124), iq2xs 9197 -> 6297 (5386). A single u16 load for iq2xs's word measured slower (6991) and stays two byte loads. 85 variants ok in gen_tune_probe TEST mode. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 24 +++++++++++---- plans/kernel_parity_pass.md | 29 ++++++++++++------- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 418190d690..2de83b9a8f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1325,6 +1325,21 @@ def private signed_grid_word(var te : TileEmit; var gidx, sgn : LLVMOpaqueValue? let b = te.builder var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gidx, "") var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "") + return sign_grid_word(te, gw, sgn, h) +} + +//! the u64 grid's word pair at even index `idx2`: one 8-byte load, half `h` split off in registers +//! (the load is shared by the h = 0 and h = 1 decodes through CSE) +def private signed_grid_pair_half(var te : TileEmit; var idx2, sgn : LLVMOpaqueValue?; h : int) : LLVMOpaqueValue? { + let b = te.builder + var gp = LLVMBuildGEP2(b, te.types.t_int64, te.iq3s_grid_g, LLVMBuildLShr(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), ""), "") + var g64 = LLVMBuildLoad2Aligned(b, te.types.t_int64, gp, 4u, "") + var gw = LLVMBuildTrunc(b, h == 0 ? g64 : LLVMBuildLShr(b, g64, LLVMConstInt(te.types.t_int64, 32ul, 0), ""), te.types.t_int32, "") + return sign_grid_word(te, gw, sgn, h) +} + +def private sign_grid_word(var te : TileEmit; var gw, sgn : LLVMOpaqueValue?; h : int) : LLVMOpaqueValue? { + let b = te.builder if (te.signVec) { return gw } @@ -1421,8 +1436,7 @@ def private decode_iq2s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; bl var iv0 = wg_byte(te, base, wg_col(mr, blk * 4 + l)) var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 0x300ul, 0), "") var idx2 = LLVMBuildShl(b, LLVMBuildOr(b, iv0, hb, ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - return signed_grid_word(te, gi, sgn, h) + return signed_grid_pair_half(te, idx2, sgn, h) } //! iq2xs: the u16 word's low 9 bits index the u64 grid's word pair, its high 7 bits the ksigns code @@ -1435,8 +1449,7 @@ def private decode_iq2xs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; b var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "") var idx2 = LLVMBuildShl(b, LLVMBuildAnd(b, w16, LLVMConstInt(te.types.t_int32, 511ul, 0), ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") var sgn = ksigns_word(te, LLVMBuildLShr(b, w16, LLVMConstInt(te.types.t_int32, 9ul, 0), "")) - var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - return signed_grid_word(te, gi, sgn, h) + return signed_grid_pair_half(te, idx2, sgn, h) } //! iq2xxs: block b's grid bytes (column 2b) index the u64 grid's word pair; its aux32 (column 2b+1) carries the ksigns codes @@ -1449,8 +1462,7 @@ def private decode_iq2xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; } var idx2 = LLVMBuildShl(b, wg_byte(te, base, 2 * blk * mr * 4 + l), LLVMConstInt(te.types.t_int32, 1ul, 0), "") var sidx = LLVMBuildAnd(b, LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), LLVMConstInt(te.types.t_int32, 127ul, 0), "") - var gi = h == 0 ? idx2 : LLVMBuildAdd(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - return signed_grid_word(te, gi, ksigns_word(te, sidx), h) + return signed_grid_pair_half(te, idx2, ksigns_word(te, sidx), h) } def private decode_kq_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index daca3c1a7a..eb255b5812 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -44,11 +44,11 @@ proven fact per push; research before any kernel edit (the two memos below). |---|---|---|---|---|---| | k4 | 1969 (8.6 ns/sb) | 2421 | 1.23x | 120.9 GFLOP/s | 71.1 GFLOP/s (1.70x) | | k2 | 2017 (8.8 ns/sb) | 2132 | 1.06x | - | - | -| iq3s | 11406 (49.7 ns/sb); sign=vec 7732 (33.7) | 10340 | 0.91x -> 1.34x | - | - | -| iq3xxs | 11578 (50.5) | 6590 | 0.57x | - | - | -| iq2s | 11732 (51.1); sign=vec 7076 (30.8) | 5074 | 0.43x -> 0.72x | - | - | -| iq2xs | 11490 (50.1) | 5386 | 0.47x | - | - | -| iq2xxs | 11061 (48.2) | 5124 | 0.46x | - | - | +| iq3s | 11406 (49.7 ns/sb); sign=vec 7562 (33.0) | 10340 | 0.91x -> 1.37x | - | - | +| iq3xxs | 11578 (50.5); sign=vec 7893 (34.4) | 6590 | 0.57x -> 0.83x | - | - | +| iq2s | 11732 (51.1); sign=vec + u64 pair 5152 (22.5) | 5074 | 0.43x -> 0.98x | - | - | +| iq2xs | 11490 (50.1); sign=vec + u64 pair 6297 (27.5) | 5386 | 0.47x -> 0.86x | - | - | +| iq2xxs | 11061 (48.2); sign=vec + u64 pair 5209 (22.7) | 5124 | 0.46x -> 0.98x | - | - | Reading: the five grid formats cost 48-51 ns per superblock regardless of what each decodes, against 8.6-8.8 for k4/k2. A flat cost independent of the format is a shared mechanism, not five @@ -87,11 +87,17 @@ bandwidth; on the 1B vehicles it shows, on a 27B it mostly does not - but the ke CPU decode (gap 2): 1. DONE-KILLED `gather="reg"` (measured 1.85x slower - see the ledger). -2. DONE for iq3s/iq2s: `sign="vec"` (the plane-resident sign column, one masked negate per vector). -3. Repack-baked sign bytes for iq3xxs/iq2xs/iq2xxs (the memo's tier 4) so they join `sign="vec"`; - PACK_VERSION bump. 4. The iq2 word-pair index work - a u64 grid entry decoded as two dwords is - twice llama.cpp's index work per 8 weights (the remaining iq2s gap). 5. retro audit of - IQ4_XS/Q3_K per followup 60 (already >= 1.0x; low priority). +2. DONE all five: `sign="vec"` - the sign-byte column negates whole vectors; iq3s/iq2s load it + off the plane, iq3xxs/iq2xs/iq2xxs synthesize it from the 7-bit codes (parity = the 8th bit), + so no plane layout change (the repack-baked byte was killed by the plane map: grp bytes are the + disk bytes, ~30 CPU/Vulkan/Metal sites read them). +3. DONE: the iq2 formats' u64 grid pair as one 8-byte load, half split in registers. +4. OPEN: the tuner cannot crown a gemv-only spelling - it races the TILE, where sign=vec is inert + (--tune-only iq3sq8_tile_gen crowned the old seat by a tie). Decision pending the M1 numbers: + make sign=vec THE gemv path (delete the knob and its 20 seats) if it wins on ARM too. +5. iq2xs 0.86x / iq3xxs 0.83x residue: the per-dword qs byte loads (a 4-byte column load with + in-register byte extraction is the next candidate); the u16 single load for iq2xs measured + SLOWER (6991 vs 6306 us) and was dropped. 6. retro audit of IQ4_XS/Q3_K per followup 60. Vulkan pp (gap 1): 1. Budget split (measurement, no kernel edit) - is the GEMM the 30%? 2. `shAscales`-style scale @@ -111,5 +117,8 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. One thread, m=4096 k=14336: iq3s 11581 -> 7732 us (1.34x of the reference's 10340), iq2s 12413 -> 7076 us (0.72x of 5074, was 0.41x). Every variant bit-exact in gen_tune_probe TEST mode. The five gather emitters collapsed into one gather over per-format decode functions (-110 lines). +- 2026-09-01: sign=vec for iq3xxs/iq2xs/iq2xxs via the parity-synthesized column (no layout + change) and the u64 grid pair load for the three iq2 formats: iq2s 5152 us (0.98x), iq2xxs 5209 + (0.98x), iq2xs 6297 (0.86x), iq3xxs 7893 (0.83x), iq3s 7562 (1.37x); 85 variants ok in TEST mode. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From c2507a236bcd6267c8f30918e413fcd9ca1d2d0a Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:15:43 -0700 Subject: [PATCH 006/123] kernel parity, CPU item 4: the vector sign column is the grid formats' gemv path - the sign knob and its 20 seats go The tuner races the tile, where a gemv-only spelling is an identical body and ties, so --tune-only crowned the old seat; and on the M1's sdot lattice the vector column lost nowhere (iq3s 7383 -> 6105 us, iq2s 7161 -> 5012, iq2xxs 7602 -> 6293, iq2xs 7878 -> 7081, iq3xxs 7516 -> 7452, one thread). A win on every tier is not a knob: the gemv always negates per vector off the sign-byte column, the decode functions return raw grid words, the smask and ksigns globals go. Shipped profiles keep their seat names; LLVM_JIT_CODEGEN_VERSION 0x5a re-keys the JIT caches for the emission change. Plain seats on zen2: iq3s 7537, iq3xxs 7862, iq2s 5202, iq2xs 6501, iq2xxs 5121 us; 65 variants ok in TEST mode; test_kquant 265 passed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 304 ++++-------------- .../dasLLAMA/dasllama/dasllama_math_gen.das | 20 -- modules/dasLLAMA/followup_general.md | 6 +- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- plans/kernel_parity_pass.md | 11 +- 5 files changed, 71 insertions(+), 272 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 2de83b9a8f..6512e8b1c7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -90,7 +90,6 @@ struct private TilePerm { mr : int = GEMM_REFERENCE_MR() gkstep : int = 1 dotPrim : string = "sdot" - sign : string = "scalar" // the grid formats' gemv: "scalar" = signs applied per grid dword in the gather, "vec" = raw words in the panel, one masked negate per weight vector off a sign-byte column width : int = 128 bias : int = 0 // 128 = bias128: grp bytes baked as w^0x80, plain vpdpbusd dots pipe : int = 0 // 1 = software-pipelined amx fold (slice K I4): double-buffered C @@ -111,8 +110,6 @@ def private parse_perm(gc : LlvmCodeCtx) : TilePerm { if (gv is tInt) { p.gkstep = gv as tInt } let dv = find_arg(gc.ann.arguments, "dot") if (dv is tString) { p.dotPrim = dv as tString } - let sgv = find_arg(gc.ann.arguments, "sign") - if (sgv is tString) { p.sign = sgv as tString } let wv = find_arg(gc.ann.arguments, "width") if (wv is tInt) { p.width = wv as tInt } let bv = find_arg(gc.ann.arguments, "bias") @@ -128,7 +125,6 @@ def private parse_perm(gc : LlvmCodeCtx) : TilePerm { // fall back together on every rail, or repack and kernel desync (the two-function-stamp // contract). Rails: knob validity, schema, per-ISA leg, geometry, vreg budget, dot availability. def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint:STYLE037,STYLE038 — the flat decline ladder IS the perm contract; splitting hides which leg refused - if (p.sign != "scalar" && p.sign != "vec") return true if (p.dotPrim == "amx_int8") { // the TMUL leg: 2×2 TDPBSSD macro over the grp16/kg4 plane; kstep pinned 1 (Q8's per-block scale boundary); Linux-only (arch_prctl enable) if ((p.kstep != 1) @@ -250,10 +246,7 @@ struct private TileEmit { abs_ty : LLVMOpaqueType? psign_decl : LLVMOpaqueValue? // x64 width-256 sign apply (VPSIGNB) iq3s_panel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) - signVec : bool // sign="vec": the gemv panel holds raw grid words; the signs land as one masked negate per weight vector iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) - iq3s_smask_g : LLVMOpaqueValue? // iq3s gemv: the [16 x i32] sign-nibble byte-mask global (iq3xxs shares it) - iq3xxs_ksigns_g : LLVMOpaqueValue? // iq3xxs gemv: the [128 x i32] ksigns_iq2xs global (7-bit index -> sign byte) gfn : LLVMOpaqueValue? // the function under emit (block creation for emitted inner loops) gctx : LLVMOpaqueContext? psign_ty : LLVMOpaqueType? @@ -1258,9 +1251,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } -// The iq3s gemv gather globals: the grid as a [512 x i32] private constant, the 4-bit sign -// nibble expanded to a byte mask via a [16 x i32] table (negation is (w ^ m) + (m & 0x01010101); -// grid magnitudes are odd 1..15, so no carry crosses a byte lane). +// The iq3s gemv gather global: the grid as a [512 x i32] private constant. def private iq3s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq3s.grid") if (grid == null) { @@ -1277,29 +1268,9 @@ def private iq3s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetAlignment(grid, 64u) } te.iq3s_grid_g = grid - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) - } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) - } - te.iq3s_smask_g = smask } -// The five panel formats' per-dword decode - one grid word of the row whose packed grp region +// The five grid formats' per-dword decode - the raw grid word of the row whose packed grp region // starts at byte `base`, for block `blk`, sign group `l`, half `h`; loads shared across (blk, l, h) fold in LLVM's CSE. def private wg_byte(var te : TileEmit; var base : LLVMOpaqueValue?; off : int) : LLVMOpaqueValue? { var p = LLVMBuildGEP2(te.builder, te.types.t_int8, te.wg, LLVMBuildAdd(te.builder, base, te.types->ConstI64(uint64(off)), ""), "") @@ -1314,51 +1285,75 @@ def private wg_i32(var te : TileEmit; var base : LLVMOpaqueValue?; off : int) : //! a packed-plane byte position to its grp column offset (4-byte columns, mr rows each) def private wg_col(mr, pos : int) : int => (pos / 4) * mr * 4 + pos % 4 -def private ksigns_word(var te : TileEmit; var sidx : LLVMOpaqueValue?) : LLVMOpaqueValue? { - var p = LLVMBuildGEP2(te.builder, te.types.t_int32, te.iq3xxs_ksigns_g, sidx, "") - return LLVMBuildLoad2Aligned(te.builder, te.types.t_int32, p, 4u, "") -} - -//! grid[gidx] with sign nibble `h` of `sgn` applied: (g ^ m) + (m & 0x01010101) negates the masked -//! bytes. Under sign="vec" the word stays raw - the block emitter negates whole vectors instead. -def private signed_grid_word(var te : TileEmit; var gidx, sgn : LLVMOpaqueValue?; h : int) : LLVMOpaqueValue? { - let b = te.builder - var gp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_grid_g, gidx, "") - var gw = LLVMBuildLoad2Aligned(b, te.types.t_int32, gp, 4u, "") - return sign_grid_word(te, gw, sgn, h) +//! grid[gidx] - a raw magnitude word; the block emitter applies the signs per vector +def private grid_word(var te : TileEmit; var gidx : LLVMOpaqueValue?) : LLVMOpaqueValue? { + var gp = LLVMBuildGEP2(te.builder, te.types.t_int32, te.iq3s_grid_g, gidx, "") + return LLVMBuildLoad2Aligned(te.builder, te.types.t_int32, gp, 4u, "") } //! the u64 grid's word pair at even index `idx2`: one 8-byte load, half `h` split off in registers -//! (the load is shared by the h = 0 and h = 1 decodes through CSE) -def private signed_grid_pair_half(var te : TileEmit; var idx2, sgn : LLVMOpaqueValue?; h : int) : LLVMOpaqueValue? { +//! (the h = 0 and h = 1 decodes share the load through CSE) +def private grid_pair_half(var te : TileEmit; var idx2 : LLVMOpaqueValue?; h : int) : LLVMOpaqueValue? { let b = te.builder var gp = LLVMBuildGEP2(b, te.types.t_int64, te.iq3s_grid_g, LLVMBuildLShr(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), ""), "") var g64 = LLVMBuildLoad2Aligned(b, te.types.t_int64, gp, 4u, "") - var gw = LLVMBuildTrunc(b, h == 0 ? g64 : LLVMBuildLShr(b, g64, LLVMConstInt(te.types.t_int64, 32ul, 0), ""), te.types.t_int32, "") - return sign_grid_word(te, gw, sgn, h) + return LLVMBuildTrunc(b, h == 0 ? g64 : LLVMBuildLShr(b, g64, LLVMConstInt(te.types.t_int64, 32ul, 0), ""), te.types.t_int32, "") } -def private sign_grid_word(var te : TileEmit; var gw, sgn : LLVMOpaqueValue?; h : int) : LLVMOpaqueValue? { +//! iq3s: 8-bit qs byte + the block's qh bit = the 9-bit grid index +def private decode_iq3s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { let b = te.builder - if (te.signVec) { - return gw - } - let c15 = LLVMConstInt(te.types.t_int32, 15ul, 0) - var mv = h == 0 ? LLVMBuildAnd(b, sgn, c15, "") : LLVMBuildAnd(b, LLVMBuildLShr(b, sgn, LLVMConstInt(te.types.t_int32, 4ul, 0), ""), c15, "") - var mp = LLVMBuildGEP2(b, te.types.t_int32, te.iq3s_smask_g, mv, "") - var mw = LLVMBuildLoad2Aligned(b, te.types.t_int32, mp, 4u, "") - return LLVMBuildAdd(b, LLVMBuildXor(b, gw, mw, ""), LLVMBuildAnd(b, mw, LLVMConstInt(te.types.t_int32, 0x01010101ul, 0), ""), "") + let mr = te.interleave + var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) + var qv = wg_byte(te, base, wg_col(mr, blk * 8 + 2 * l + h)) + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - h - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 256ul, 0), "") + return grid_word(te, LLVMBuildOr(b, qv, hb, "")) +} + +//! iq3xxs: an 8-bit index into the halved grid +def private decode_iq3xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + return grid_word(te, wg_byte(te, base, wg_col(te.interleave, blk * 8 + 2 * l + h))) } -//! sign="vec": block `blk`'s sign bytes for rows qd*rv.., byte (r, l) at r*4 + l - the column -//! itself for iq3s/iq2s, synthesized from the 7-bit ksigns codes for the other three (the 8th -//! bit is the code's parity, ksigns[i] == i | parity(i) << 7) +//! iq2s: qs byte | the qh 0x300 window = the 10-bit index, doubled into the u64 grid's word pair +def private decode_iq2s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + let b = te.builder + let mr = te.interleave + var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) + var iv0 = wg_byte(te, base, wg_col(mr, blk * 4 + l)) + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 0x300ul, 0), "") + var idx2 = LLVMBuildShl(b, LLVMBuildOr(b, iv0, hb, ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") + return grid_pair_half(te, idx2, h) +} + +//! iq2xs: the u16 word's low 9 bits index the u64 grid's word pair (two byte loads: one u16 load measured slower) +def private decode_iq2xs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + let b = te.builder + let mr = te.interleave + let lp = (blk * 4 + l) * 2 + var qlo = wg_byte(te, base, wg_col(mr, lp)) + var qhi = wg_byte(te, base, wg_col(mr, lp) + 1) + var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "") + var idx2 = LLVMBuildShl(b, LLVMBuildAnd(b, w16, LLVMConstInt(te.types.t_int32, 511ul, 0), ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") + return grid_pair_half(te, idx2, h) +} + +//! iq2xxs: block b's grid bytes (column 2b) index the u64 grid's word pair +def private decode_iq2xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { + let b = te.builder + var idx2 = LLVMBuildShl(b, wg_byte(te, base, 2 * blk * te.interleave * 4 + l), LLVMConstInt(te.types.t_int32, 1ul, 0), "") + return grid_pair_half(te, idx2, h) +} + +//! block `blk`'s sign bytes for rows qd*rv.., byte (r, l) at r*4 + l - the plane's own column for +//! iq3s/iq2s, synthesized from the 7-bit ksigns codes for the other three (the 8th bit is the +//! code's parity, ksigns[i] == i | parity(i) << 7); one masked negate per weight vector applies them def private kq_sign_bytes(var te : TileEmit; var wbP : LLVMOpaqueValue?; blk, qd : int; name : string) : LLVMOpaqueValue? { let b = te.builder let mr = te.interleave let w8 = te.width / 8 let colBytes = uint64(mr * 4) - if (te.kq == 33 || te.kq == 23) { + if (te.kq == 33 || te.kq == 23) { // the plane carries the sign bytes let col = (te.kq == 33 ? 18 : 8) + blk return load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(col) * colBytes + uint64(qd * w8)), ""), name) } @@ -1406,65 +1401,6 @@ def private apply_sign_col(var te : TileEmit; var w, col : LLVMOpaqueValue?; l, return LLVMBuildSub(b, LLVMBuildXor(b, w, m, ""), m, name) } -//! iq3s: 8-bit qs byte + the block's qh bit = the 9-bit grid index; explicit sign byte per 8 weights -def private decode_iq3s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { - let b = te.builder - let mr = te.interleave - var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) - var sgn = wg_byte(te, base, wg_col(mr, 72 + blk * 4 + l)) - var qv = wg_byte(te, base, wg_col(mr, blk * 8 + 2 * l + h)) - var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - h - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 256ul, 0), "") - return signed_grid_word(te, LLVMBuildOr(b, qv, hb, ""), sgn, h) -} - -//! iq3xxs: 8-bit index into the halved grid; the block's aux32 carries four 7-bit ksigns codes -def private decode_iq3xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { - let b = te.builder - let mr = te.interleave - var aux = wg_i32(te, base, (16 + blk) * mr * 4) - var sidx = LLVMBuildAnd(b, l == 0 ? aux : LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), LLVMConstInt(te.types.t_int32, 127ul, 0), "") - var iv = wg_byte(te, base, wg_col(mr, blk * 8 + 2 * l + h)) - return signed_grid_word(te, iv, ksigns_word(te, sidx), h) -} - -//! iq2s: qs byte | the qh 0x300 window = the 10-bit index, doubled into the u64 grid's word pair; explicit sign byte -def private decode_iq2s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { - let b = te.builder - let mr = te.interleave - var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) - var sgn = wg_byte(te, base, wg_col(mr, 32 + blk * 4 + l)) - var iv0 = wg_byte(te, base, wg_col(mr, blk * 4 + l)) - var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 0x300ul, 0), "") - var idx2 = LLVMBuildShl(b, LLVMBuildOr(b, iv0, hb, ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") - return signed_grid_pair_half(te, idx2, sgn, h) -} - -//! iq2xs: the u16 word's low 9 bits index the u64 grid's word pair, its high 7 bits the ksigns code -def private decode_iq2xs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { - let b = te.builder - let mr = te.interleave - let lp = (blk * 4 + l) * 2 - var qlo = wg_byte(te, base, wg_col(mr, lp)) - var qhi = wg_byte(te, base, wg_col(mr, lp) + 1) - var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "") - var idx2 = LLVMBuildShl(b, LLVMBuildAnd(b, w16, LLVMConstInt(te.types.t_int32, 511ul, 0), ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var sgn = ksigns_word(te, LLVMBuildLShr(b, w16, LLVMConstInt(te.types.t_int32, 9ul, 0), "")) - return signed_grid_pair_half(te, idx2, sgn, h) -} - -//! iq2xxs: block b's grid bytes (column 2b) index the u64 grid's word pair; its aux32 (column 2b+1) carries the ksigns codes -def private decode_iq2xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { - let b = te.builder - let mr = te.interleave - var aux = wg_byte(te, base, (2 * blk + 1) * mr * 4) - for (k in range(1, 4)) { - aux = LLVMBuildOr(b, aux, LLVMBuildShl(b, wg_byte(te, base, (2 * blk + 1) * mr * 4 + k), LLVMConstInt(te.types.t_int32, uint64(8 * k), 0), ""), "") - } - var idx2 = LLVMBuildShl(b, wg_byte(te, base, 2 * blk * mr * 4 + l), LLVMConstInt(te.types.t_int32, 1ul, 0), "") - var sidx = LLVMBuildAnd(b, LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), ""), LLVMConstInt(te.types.t_int32, 127ul, 0), "") - return signed_grid_pair_half(te, idx2, ksigns_word(te, sidx), h) -} - def private decode_kq_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { if (te.kq == 33) return decode_iq3s_dword(te, base, blk, l, h) if (te.kq == 34) return decode_iq3xxs_dword(te, base, blk, l, h) @@ -1537,41 +1473,6 @@ def private iq3xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetAlignment(grid, 64u) } te.iq3s_grid_g = grid - var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") - if (ksg == null) { - let kt = ksigns_iq2xs() - var elems3 : array - elems3 |> reserve(128) - for (i in range(128)) { - elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) - } - ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") - LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) - LLVMSetGlobalConstant(ksg, 1) - LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(ksg, 16u) - } - te.iq3xxs_ksigns_g = ksg - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) - } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) - } - te.iq3s_smask_g = smask } @@ -1594,26 +1495,6 @@ def private iq2s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetAlignment(grid, 64u) } te.iq3s_grid_g = grid - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) - } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) - } - te.iq3s_smask_g = smask } @@ -1635,41 +1516,6 @@ def private iq2xs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetAlignment(grid, 64u) } te.iq3s_grid_g = grid - var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") - if (ksg == null) { - let kt = ksigns_iq2xs() - var elems3 : array - elems3 |> reserve(128) - for (i in range(128)) { - elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) - } - ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") - LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) - LLVMSetGlobalConstant(ksg, 1) - LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(ksg, 16u) - } - te.iq3xxs_ksigns_g = ksg - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) - } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) - } - te.iq3s_smask_g = smask } // The iq2xxs gemv gather globals: the u64 grid as a [512 x i32] private constant (256 entries @@ -1690,41 +1536,6 @@ def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetAlignment(grid, 64u) } te.iq3s_grid_g = grid - var ksg = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.ksigns") - if (ksg == null) { - let kt = ksigns_iq2xs() - var elems3 : array - elems3 |> reserve(128) - for (i in range(128)) { - elems3 |> push(LLVMConstInt(te.types.t_int32, uint64(kt[i]), 0)) - } - ksg = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 128u), "dasllama.iq3xxs.ksigns") - LLVMSetInitializer(ksg, LLVMConstArray(te.types.t_int32, array_data_ptr(elems3), 128u)) - LLVMSetGlobalConstant(ksg, 1) - LLVMSetLinkage(ksg, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(ksg, 16u) - } - te.iq3xxs_ksigns_g = ksg - var smask = LLVMGetNamedGlobal(mod, "dasllama.iq3s.smask") - if (smask == null) { - var elems2 : array - elems2 |> reserve(16) - for (m in range(16)) { - var v = 0ul - for (t in range(4)) { - if ((m & (1 << t)) != 0) { - v |= 0xFFul << uint64(8 * t) - } - } - elems2 |> push(LLVMConstInt(te.types.t_int32, v, 0)) - } - smask = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int32, 16u), "dasllama.iq3s.smask") - LLVMSetInitializer(smask, LLVMConstArray(te.types.t_int32, array_data_ptr(elems2), 16u)) - LLVMSetGlobalConstant(smask, 1) - LLVMSetLinkage(smask, LLVMLinkage.LLVMPrivateLinkage) - LLVMSetAlignment(smask, 16u) - } - te.iq3s_smask_g = smask } @@ -1788,8 +1599,8 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var a[i] = LLVMConstNull(te.vni32) a1[i] = LLVMConstNull(te.vni32) } - var signCol : LLVMOpaqueValue? [2] // sign="vec": this block's sign bytes, rows qd*rv.. at stride 4, one column load per vector row-band - if (gather && te.signVec) { + var signCol : LLVMOpaqueValue? [2] // this block's sign bytes, rows qd*rv.. at stride 4 - the gemv negates per vector, the tile's panel arrives signed + if (gather) { for (qd in range(rq)) { signCol[qd] = kq_sign_bytes(te, wbP, blk, qd, "sgc{blk}_{qd}") } @@ -2059,7 +1870,6 @@ def private setup_tile_emit(var te : TileEmit; var gc : LlvmCodeCtx; p : TilePer te.kgroup = (te.mx4 || te.q51 || te.kq != 0) ? 4 : rt.kgroup // the perm's bias applies to the Q8 weight plane only — mx4/kq/q51 planes never see the biased repack te.bias = (te.mx4 || te.q51 || te.kq != 0) ? 0 : p.bias - te.signVec = p.sign == "vec" te.width = (te.dotKind == DOT_SDOT || te.dotKind == DOT_SMMLA) ? 128 : p.width te.rv = te.width / 32 te.rq = rt.interleave / te.rv diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 7342a271cb..27f4e2932d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -690,10 +690,6 @@ def iq3sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(mr = 8, sign = "vec"), - tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq3sq8_gemv_gen", gen = "dasllama_gemm_gen::iq3s_gemv"), tune_companion(fn = "iq3sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3s_tile", @@ -769,10 +765,6 @@ def iq3xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(mr = 8, sign = "vec"), - tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq3xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq3xxs_gemv"), tune_companion(fn = "iq3xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3xxs_tile", @@ -977,10 +969,6 @@ def iq2sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(mr = 8, sign = "vec"), - tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq2sq8_gemv_gen", gen = "dasllama_gemm_gen::iq2s_gemv"), tune_companion(fn = "iq2sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2s_tile", @@ -1052,10 +1040,6 @@ def iq2xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(mr = 8, sign = "vec"), - tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq2xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xs_gemv"), tune_companion(fn = "iq2xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xs_tile", @@ -1129,10 +1113,6 @@ def iq2xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(mr = 8, sign = "vec"), - tune_perm(dot = "maddubs", width = 256, mr = 8, sign = "vec", requires = "avx2"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 8, sign = "vec", requires = "avxvnni|avx512vnni"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, sign = "vec", requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq2xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xxs_gemv"), tune_companion(fn = "iq2xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xxs_tile", diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index d651d8c0d5..037006eb51 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -777,7 +777,11 @@ crowned by the probe; plan and fact base: `plans/kernel_parity_pass.md`. 2026-09-01: `gather="reg"` measured 1.85x SLOWER (insertelement chains, 5x the code) and was dropped; `sign="vec"` landed for iq3s (7732 us = 1.34x the reference) and iq2s (7076 us = - 0.72x). iq3xxs/iq2xs/iq2xxs join once the repack bakes their sign bytes into the plane. + 0.72x). Then all five: the sign column synthesized from the 7-bit codes (parity = the 8th bit, + no plane change), the iq2 formats' u64 grid pair as one load, and the knob collapsed into the + one gemv path (the tuner races the tile, where a gemv-only spelling ties). Kernel-level, one + thread: iq3s 1.37x, iq2s 0.98x, iq2xxs 0.98x, iq2xs 0.86x, iq3xxs 0.83x of the reference. + Open residue: the per-dword qs byte loads of iq2xs/iq3xxs. 62. **IQ3_S Metal decode: the ~140 GB/s compose ceiling (tg 0.95x).** Eight GEMV forms raced at n=2048 d=8192 - gather placement x3, gather deleted, signs deleted, llama.cpp's exact diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index ca7780097e..721baab44a 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x59ul // aarch64 hosts append +i8mm when cpu_supports says so (0x58: [hint(unsafe_division_check)] drops the sdiv/srem guards) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5aul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index eb255b5812..4071d943a6 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -92,9 +92,11 @@ CPU decode (gap 2): so no plane layout change (the repack-baked byte was killed by the plane map: grp bytes are the disk bytes, ~30 CPU/Vulkan/Metal sites read them). 3. DONE: the iq2 formats' u64 grid pair as one 8-byte load, half split in registers. -4. OPEN: the tuner cannot crown a gemv-only spelling - it races the TILE, where sign=vec is inert - (--tune-only iq3sq8_tile_gen crowned the old seat by a tie). Decision pending the M1 numbers: - make sign=vec THE gemv path (delete the knob and its 20 seats) if it wins on ARM too. +4. DONE: the vector sign column IS the gemv path - the `sign` knob and its 20 seats are gone. The + tuner races the TILE (a gemv-only spelling ties there; `--tune-only iq3sq8_tile_gen` crowned the + old seat), and the M1 (sdot lattice, one thread) showed no losses: iq3s 7383 -> 6105 us, iq2s + 7161 -> 5012, iq2xxs 7602 -> 6293, iq2xs 7878 -> 7081, iq3xxs 7516 -> 7452. Shipped profiles + stay valid (same seat names); LLVM_JIT_CODEGEN_VERSION 0x5a re-keys the JIT caches. 5. iq2xs 0.86x / iq3xxs 0.83x residue: the per-dword qs byte loads (a 4-byte column load with in-register byte extraction is the next candidate); the u16 single load for iq2xs measured SLOWER (6991 vs 6306 us) and was dropped. 6. retro audit of IQ4_XS/Q3_K per followup 60. @@ -117,6 +119,9 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. One thread, m=4096 k=14336: iq3s 11581 -> 7732 us (1.34x of the reference's 10340), iq2s 12413 -> 7076 us (0.72x of 5074, was 0.41x). Every variant bit-exact in gen_tune_probe TEST mode. The five gather emitters collapsed into one gather over per-format decode functions (-110 lines). +- 2026-09-01: the sign knob collapsed into the one gemv path (-268/+58 lines in the emitter, the + smask/ksigns globals gone); the plain seats now measure iq3s 7537, iq3xxs 7862, iq2s 5202, iq2xs + 6501, iq2xxs 5121 us; 65 variants ok in TEST mode; test_kquant 265 passed. - 2026-09-01: sign=vec for iq3xxs/iq2xs/iq2xxs via the parity-synthesized column (no layout change) and the u64 grid pair load for the three iq2 formats: iq2s 5152 us (0.98x), iq2xxs 5209 (0.98x), iq2xs 6297 (0.86x), iq3xxs 7893 (0.83x), iq3s 7562 (1.37x); 85 variants ok in TEST mode. From edaba28d974e4ba2617cb4518e5f6349bfd04c57 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 01:20:43 -0700 Subject: [PATCH 007/123] kernel parity, CPU item 5: the column dword read for iq3xxs's grid bytes and iq2xs's words One aligned dword load per 4-byte column with the byte split off in registers, the column shared through CSE. Measured per format at one thread, m=4096 k=14336: iq3xxs 7862 -> 6602 us (reference 6590) and iq2xs 6501 -> 4702 (5386) win; iq3s 7537 -> 8671, iq2s 5202 -> 6187 and iq2xxs 5121 -> 5574 lose and keep their byte loads. 65 variants ok in gen_tune_probe TEST mode. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 21 +++++++++++++----- plans/kernel_parity_pass.md | 22 +++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 6512e8b1c7..7a8eb6d1a9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1285,6 +1285,16 @@ def private wg_i32(var te : TileEmit; var base : LLVMOpaqueValue?; off : int) : //! a packed-plane byte position to its grp column offset (4-byte columns, mr rows each) def private wg_col(mr, pos : int) : int => (pos / 4) * mr * 4 + pos % 4 +//! the byte at packed position `pos`, read as one dword load of its column with the byte split off in +//! registers - the four positions of a column share the load through CSE +def private wg_col_byte(var te : TileEmit; var base : LLVMOpaqueValue?; pos : int) : LLVMOpaqueValue? { + let b = te.builder + var col = wg_i32(te, base, (pos / 4) * te.interleave * 4) + let sh = 8 * (pos % 4) + var v = sh == 0 ? col : LLVMBuildLShr(b, col, LLVMConstInt(te.types.t_int32, uint64(sh), 0), "") + return sh == 24 ? v : LLVMBuildAnd(b, v, LLVMConstInt(te.types.t_int32, 255ul, 0), "") +} + //! grid[gidx] - a raw magnitude word; the block emitter applies the signs per vector def private grid_word(var te : TileEmit; var gidx : LLVMOpaqueValue?) : LLVMOpaqueValue? { var gp = LLVMBuildGEP2(te.builder, te.types.t_int32, te.iq3s_grid_g, gidx, "") @@ -1310,9 +1320,9 @@ def private decode_iq3s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; bl return grid_word(te, LLVMBuildOr(b, qv, hb, "")) } -//! iq3xxs: an 8-bit index into the halved grid +//! iq3xxs: an 8-bit index into the halved grid (the column read measured 7862 -> 6602 us; byte loads stay for iq3s/iq2s/iq2xxs, where it lost) def private decode_iq3xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { - return grid_word(te, wg_byte(te, base, wg_col(te.interleave, blk * 8 + 2 * l + h))) + return grid_word(te, wg_col_byte(te, base, blk * 8 + 2 * l + h)) } //! iq2s: qs byte | the qh 0x300 window = the 10-bit index, doubled into the u64 grid's word pair @@ -1326,13 +1336,12 @@ def private decode_iq2s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; bl return grid_pair_half(te, idx2, h) } -//! iq2xs: the u16 word's low 9 bits index the u64 grid's word pair (two byte loads: one u16 load measured slower) +//! iq2xs: the u16 word's low 9 bits index the u64 grid's word pair (the column read measured 6501 -> 4702 us; one u16 load measured slower than two bytes) def private decode_iq2xs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { let b = te.builder - let mr = te.interleave let lp = (blk * 4 + l) * 2 - var qlo = wg_byte(te, base, wg_col(mr, lp)) - var qhi = wg_byte(te, base, wg_col(mr, lp) + 1) + var qlo = wg_col_byte(te, base, lp) + var qhi = wg_col_byte(te, base, lp + 1) var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "") var idx2 = LLVMBuildShl(b, LLVMBuildAnd(b, w16, LLVMConstInt(te.types.t_int32, 511ul, 0), ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") return grid_pair_half(te, idx2, h) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 4071d943a6..c5da565879 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -45,10 +45,10 @@ proven fact per push; research before any kernel edit (the two memos below). | k4 | 1969 (8.6 ns/sb) | 2421 | 1.23x | 120.9 GFLOP/s | 71.1 GFLOP/s (1.70x) | | k2 | 2017 (8.8 ns/sb) | 2132 | 1.06x | - | - | | iq3s | 11406 (49.7 ns/sb); sign=vec 7562 (33.0) | 10340 | 0.91x -> 1.37x | - | - | -| iq3xxs | 11578 (50.5); sign=vec 7893 (34.4) | 6590 | 0.57x -> 0.83x | - | - | -| iq2s | 11732 (51.1); sign=vec + u64 pair 5152 (22.5) | 5074 | 0.43x -> 0.98x | - | - | -| iq2xs | 11490 (50.1); sign=vec + u64 pair 6297 (27.5) | 5386 | 0.47x -> 0.86x | - | - | -| iq2xxs | 11061 (48.2); sign=vec + u64 pair 5209 (22.7) | 5124 | 0.46x -> 0.98x | - | - | +| iq3xxs | 11578 (50.5); sign column + column read 6641 (29.0) | 6590 | 0.57x -> 0.99x | - | - | +| iq2s | 11732 (51.1); sign column + u64 pair 5039-5202 | 5074 | 0.43x -> 0.98-1.01x | - | - | +| iq2xs | 11490 (50.1); sign column + u64 pair + column read 4831 (21.1) | 5386 | 0.47x -> 1.11x | - | - | +| iq2xxs | 11061 (48.2); sign column + u64 pair 5121-5487 (noise band) | 5124 | 0.46x -> 0.93-1.00x | - | - | Reading: the five grid formats cost 48-51 ns per superblock regardless of what each decodes, against 8.6-8.8 for k4/k2. A flat cost independent of the format is a shared mechanism, not five @@ -97,9 +97,14 @@ CPU decode (gap 2): old seat), and the M1 (sdot lattice, one thread) showed no losses: iq3s 7383 -> 6105 us, iq2s 7161 -> 5012, iq2xxs 7602 -> 6293, iq2xs 7878 -> 7081, iq3xxs 7516 -> 7452. Shipped profiles stay valid (same seat names); LLVM_JIT_CODEGEN_VERSION 0x5a re-keys the JIT caches. -5. iq2xs 0.86x / iq3xxs 0.83x residue: the per-dword qs byte loads (a 4-byte column load with - in-register byte extraction is the next candidate); the u16 single load for iq2xs measured - SLOWER (6991 vs 6306 us) and was dropped. 6. retro audit of IQ4_XS/Q3_K per followup 60. +5. DONE where it won: the column dword read with in-register byte extraction - iq3xxs 7862 -> + 6602 us and iq2xs 6501 -> 4702; it LOST on iq3s (7537 -> 8671), iq2s (5202 -> 6187) and iq2xxs + (5121 -> 5574), which keep byte loads. The u16 single load for iq2xs measured slower (6991 vs + 6306) and was dropped. +6. Residue at one thread: iq3xxs 0.99x, iq2s 0.98-1.01x, iq2xxs 0.93-1.00x (the noise band of the + 1-thread bench is ~7%; a longer run or the 16-thread engine shape decides the last 2%). 7. the + 16-thread stamp on the vehicles (needs `--for-debug-purposes` on lcpp_bench, or the released + exe). 8. retro audit of IQ4_XS/Q3_K per followup 60. Vulkan pp (gap 1): 1. Budget split (measurement, no kernel edit) - is the GEMM the 30%? 2. `shAscales`-style scale @@ -119,6 +124,9 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. One thread, m=4096 k=14336: iq3s 11581 -> 7732 us (1.34x of the reference's 10340), iq2s 12413 -> 7076 us (0.72x of 5074, was 0.41x). Every variant bit-exact in gen_tune_probe TEST mode. The five gather emitters collapsed into one gather over per-format decode functions (-110 lines). +- 2026-09-01: the column dword read lands for iq3xxs and iq2xs only (measured per format, see the + queue); zen2 one thread now: iq3s 7673, iq3xxs 6641, iq2s 5039-5174, iq2xs 4831, iq2xxs + 5281-5487 us; 65 variants ok in TEST mode. - 2026-09-01: the sign knob collapsed into the one gemv path (-268/+58 lines in the emitter, the smask/ksigns globals gone); the plain seats now measure iq3s 7537, iq3xxs 7862, iq2s 5202, iq2xs 6501, iq2xxs 5121 us; 65 variants ok in TEST mode; test_kquant 265 passed. From 714eb8f5eafd7abd10fadd7a2303eb2c2a1f7762 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 02:03:55 -0700 Subject: [PATCH 008/123] kernel parity: the lab covers all sixteen formats - kq_kernel_bench arms for q8, mxfp4 and q5_1, interleaved rounds, and harness/kernel_ladder.sh The bench times every format's registry rows at one thread on synthetic planes with the rows of a format interleaved round-robin (best and median reported); q8 and its mx4 companion ride the q8q8 grid's registries with the tokstep companion's tokens per call, q51 its per-32 planes, and the tile arm hands k5, k6 and the five grid formats the byte-expanded panel they read (kq_reads_packed_planes decides). kernel_ladder.sh runs the bench and the reference exe's test-backend-ops perf at the same shape for every format and prints the box's ratio table; it resolves its tree from its own path, never the global DASLANG. The zen2 table and the 16-thread vehicle rows land in plans/kernel_parity_pass.md and the HOW_TO sections. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 14 +- .../benchmarks/matmul/kq_kernel_bench.das | 400 +++++++++++++----- modules/dasLLAMA/harness/README.md | 1 + modules/dasLLAMA/harness/kernel_ladder.sh | 75 ++++ plans/kernel_parity_pass.md | 43 +- 5 files changed, 431 insertions(+), 102 deletions(-) create mode 100644 modules/dasLLAMA/harness/kernel_ladder.sh diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index e2fd0b5c90..48dae6595b 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -265,8 +265,9 @@ invalidation ledger: `plans/jit_compile_time.md`. modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das -- --fmt --perm ` times every row of the tile's and gemv's `_variants()` registries at one thread on synthetic planes (seconds per try), and the reference row at the same shape is the reference exe's `test-backend-ops perf --o MUL_MAT -p "type_a=,type_b=f32,m=4096,n=1,"` under `GGML_BENCH_THREADS=1`. The app run -comes only after a spelling wins there. Procedure, fact base and work queue: +-o MUL_MAT -p "type_a=,type_b=f32,m=4096,n=1,"` under `GGML_BENCH_THREADS=1`; +`harness/kernel_ladder.sh` runs both sides for every format and prints the box's ratio table. The +app run comes only after a spelling wins there. Procedure, fact base and work queue: `plans/kernel_parity_pass.md`. A real file whose every tensor type is now loadable (the header census script in the session @@ -563,6 +564,9 @@ only the step-3 0.0267 top-2 flip vs llama.cpp. M1 benches: CPU das 897.4/56.3 v is CLOSED - and with it THE FORMAT LADDER: four-tier table zen2 2.86x/0.70x, vk 0.78x/0.70x, M1 CPU 6.41x/0.57x, Metal 0.93x/0.78x. +2026-09-01 (kernel parity pass): zen2 16t vs clean-cpu on the IQ2_XXS-local 1B: pp512 506.9 vs +181.2 (2.80x), tg128 88.6 vs 86.2 (1.03x, was 0.70x) - the sign column + u64 pair gemv. + ### IQ2_XS Phase A (CPU, 2026-08-31) - the ksigns u64 tier Shape: 256-superblock grid format - each of the 32 u16 qs words carries a 9-bit index into @@ -630,6 +634,9 @@ tiers. M1 16GB benches: CPU das 746.6/51.6 vs llama.cpp 144.8/101.1 (5.16x/0.51x (0.93x/0.86x - the iq2s pp class). The format is CLOSED on all four tiers; four-tier table: zen2 2.78x/0.70x, vk 0.77x/0.54x, M1 CPU 5.16x/0.51x, Metal 0.93x/0.86x. +2026-09-01 (kernel parity pass): zen2 16t vs clean-cpu on the IQ2_XS-local 1B: pp512 480.7 vs +174.6 (2.75x), tg128 93.8 vs 85.9 (1.09x, was 0.70x) - sign column + u64 pair + column dword read. + ### IQ2_S Phase A (CPU, 2026-08-31) - the u64-grid tier Shape: 256-superblock grid format, the first with a u64 grid - a 10-bit index (qs byte | @@ -708,6 +715,7 @@ the mradermacher i1 vehicle - IQ2_S attn x32 + IQ3_XXS/IQ3_S/Q4_K/Q5_K): | tier | pp512 das / llama.cpp | tg128 das / llama.cpp | |---|---|---| | zen2 CPU | 501.5 / 138.5 (3.62x) | 55.8 / 73.5 (0.76x) | +| zen2 CPU, 2026-09-01 kernel parity pass | 494.4 / 138.1 (3.58x) | 71.9 / 74.5 (0.97x, +-4.2) - sign column + u64 pair | | 5060 Ti Vulkan | 12099.3 / 17377.5 (0.70x - the tier class) | 292.4 / 362.5 (0.81x) | | M1 CPU | 883.3 / 413.4 (2.14x) | 53.7 / 73.9 (0.73x) | | M1 Metal | 3170.5 / 3427.2 (0.93x) | 205.8 / 220.7 (0.93x) | @@ -924,6 +932,7 @@ the local --tensor-type requant, iq3_xxs on attn_k/q + all ffn): | tier | pp512 das / llama.cpp | tg128 das / llama.cpp | |---|---|---| | zen2 CPU | 507.2 / 136.0 (3.73x) | 56.7 / 72.6 (0.78x) | +| zen2 CPU, 2026-09-01 kernel parity pass | 501.0 / 135.8 (3.69x) | 74.9 / 74.3 (1.01x) - sign column + column dword read | | 5060 Ti Vulkan | 12225.7 / 17807.7 (0.69x - the tier class) | 372.1 / 389.9 (0.95x) | | M1 CPU | 906.0 / 410.5 (2.21x) | 53.5 / 74.0 (0.72x) | | M1 Metal | 3224.0 / 3429.9 (0.94x) | 213.5 / 227.3 (0.94x) | @@ -1026,6 +1035,7 @@ embedding head is Q6_K - three formats share every decode step): | tier | pp512 das / llama.cpp | tg128 das / llama.cpp | |---|---|---| | zen2 CPU | 516.9 / 104.9 (4.93x) | 52.4 / 57.0 (0.92x) | +| zen2 CPU, 2026-09-01 kernel parity pass | 526.1 / 104.7 (5.03x) | 69.9 / 56.5 (1.24x) - the vector sign column gemv | | 5060 Ti Vulkan | 12539.6 / 17865 (0.70x) | 288.1 / 324.2 (0.89x) | | M1 CPU | 886.2 / 433.6 (2.04x) | 57.4 / 66.6 (0.86x) | | M1 Metal | 3237.6 / 3344.3 (0.97x) | 199.4 / 209.0 (0.95x) | diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 06895e37fe..a477d6542e 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -5,27 +5,30 @@ options _dasllama_internal = true require daslib/clargs require daslib/enum_trait +require daslib/fio require dasllama/dasllama_math_gen require dasllama/dasllama_math require dasllama/dasllama_kqformat -require daslib/fio +require dasllama/dasllama_gemm_schema require math require strings require daslib/strings_boost -//! The kq kernel microbench - the inner loop of kernel work. Raw kernels, one thread, no jobque, +//! The kernel microbench - the inner loop of kernel work. Raw kernels, one thread, no jobque, //! no model; synthetic planes, because the timing is data-independent. GEMV is the decode shape -//! (one token), TILE the prefill batch. Every row of a kernel's `_variants()` registry is timed: -//! under DAS_TUNE_MODE=tune that is the whole [tune_perm] grid, under normal mode the stamped -//! winner alone. Timing only - a spelling's correctness is gen_tune_probe's TEST mode. +//! (one token), TILE the prefill batch. Every row of a kernel's `_variants()` registry is timed, +//! the rows of one format interleaved round-robin so frequency drift cancels: under +//! DAS_TUNE_MODE=tune that is the whole [tune_perm] grid, under normal mode the stamped winner +//! alone. Timing only - a spelling's correctness is gen_tune_probe's TEST mode. //! Run: DAS_TUNE_MODE=tune bin/daslang -jit modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das -- --fmt iq3s //! The reference row at the same shape: GGML_BENCH_THREADS=1 test-backend-ops perf -o MUL_MAT -p "type_a=iq3_s" -//! (its default MUL_MAT perf case is m=4096 rows, k=14336, n in {1, 512} tokens - this bench's defaults). +//! (its default MUL_MAT perf case is m=4096 rows, k=14336, n in {1, 512} tokens - this bench's defaults); +//! harness/kernel_ladder.sh runs both sides for every format and joins the table. [CommandLineArgs] struct BenchArgs { @clarg_short = "f" - @clarg_doc = "KqFmt names, comma-separated (k4,k5,k6,q40,iq4xs,k3,iq3s,iq3xxs,iq4nl,k2,iq2s,iq2xs,iq2xxs), or all" + @clarg_doc = "KqFmt names, comma-separated (q8,k4,k5,k6,q40,q51,iq4xs,k3,iq3s,iq3xxs,iq4nl,k2,iq2s,iq2xs,iq2xxs; mx4 = mxfp4), or all" fmt : string = "all" @clarg_doc = "Reduction length - the weight row length (k in test-backend-ops terms)" n : int = 14336 @@ -33,7 +36,7 @@ struct BenchArgs { d : int = 4096 @clarg_doc = "Tokens for the TILE arm (the prefill batch); 0 skips the arm" ntok : int = 512 - @clarg_doc = "Timed repetitions per row; the best one is reported" + @clarg_doc = "Timed rounds per row, interleaved across the format's rows; best and median are reported" rounds : int = 5 @clarg_doc = "Skip the GEMV (decode) arm" no_gemv : bool @@ -46,9 +49,13 @@ struct BenchArgs { typedef GemvFn = function<(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void> typedef TileFn = function<(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void> +typedef GemvQ8Fn = function<(var yp : float?; wp : int8 const?; sp : float const?; xqp : int8 const?; xsp : float const?; n, rb, re : int64) : void> +typedef TileQ8Fn = function<(var yp : float?; wg : int8 const?; sg : float const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void> +typedef GemvMx4Fn = function<(var yp : float?; wn : uint8 const?; we : uint8 const?; xqp : int8 const?; xsp : float const?; n, rb, re : int64) : void> typedef LayoutFn = function<() : int> -let TILE_TOKENS = 4l // tokens one kq tile call covers +let TILE_TOKENS = 4l // tokens one kq / q51 / mx4 tile call covers (q8's tokstep companion says its own) +let MX4_NAME = "mx4" // mxfp4 has no KqFmt row; it rides the q8q8 grid as a companion def fmt_of_name(name : string) : KqFmt { for (f in type) { @@ -74,6 +81,7 @@ def gemv_variants(f : KqFmt) : array> { if (f == KqFmt.iq2s) return <- iq2sq8_gemv_gen_variants() if (f == KqFmt.iq2xs) return <- iq2xsq8_gemv_gen_variants() if (f == KqFmt.iq2xxs) return <- iq2xxsq8_gemv_gen_variants() + if (f == KqFmt.q51) return <- q51q8_gemv_gen_variants() var none : array> return <- none } @@ -92,6 +100,7 @@ def tile_variants(f : KqFmt) : array> { if (f == KqFmt.iq2s) return <- iq2sq8_tile_gen_variants() if (f == KqFmt.iq2xs) return <- iq2xsq8_tile_gen_variants() if (f == KqFmt.iq2xxs) return <- iq2xxsq8_tile_gen_variants() + if (f == KqFmt.q51) return <- q51q8_tile_gen_variants() var none : array> return <- none } @@ -110,24 +119,41 @@ def layout_variants(f : KqFmt) : array> { if (f == KqFmt.iq2s) return <- iq2sq8_layout_gen_variants() if (f == KqFmt.iq2xs) return <- iq2xsq8_layout_gen_variants() if (f == KqFmt.iq2xxs) return <- iq2xxsq8_layout_gen_variants() - var none : array> - return <- none + if (f == KqFmt.q51) return <- q51q8_layout_gen_variants() + return <- q8q8_layout_gen_variants() // q8 and its mx4 companion share the q8q8 layout } -//! mr per registry suffix; the layout companion is stamped in lockstep with the tile, so a tile -//! row and the layout row of the same suffix belong together -def layout_mrs(f : KqFmt) : table { - var mrs : table - var lvs <- layout_variants(f) - for (v in lvs) { - mrs[v._0] = int64(invoke(v._1)) - } +//! an int-valued companion registry (layout mr, q8's tokstep) keyed by suffix; the companion is +//! stamped in lockstep with its tile, so the row of the same suffix belongs to it +def companion_ints(var lvs : array>) : table { + var vals <- {for (v in lvs); v._0 => int64(invoke(v._1))} delete lvs - return <- mrs + return <- vals } -//! the five grid formats' TILE reads a caller-unpacked panel (256 bytes per row per superblock), not the packed plane the gemv gathers from -def tile_kq_bytes_per_sb(f : KqFmt) : int64 => (f == KqFmt.iq3s || f == KqFmt.iq3xxs || f == KqFmt.iq2s || f == KqFmt.iq2xs || f == KqFmt.iq2xxs) ? 256l : kq_qsb(f) +//! a TILE reads either the packed quant plane or a caller-unpacked byte panel (256 bytes per row per superblock: k5, k6 and the five grid formats) - kq_reads_packed_planes is the truth +def tile_kq_bytes_per_sb(f : KqFmt) : int64 => kq_reads_packed_planes(int64(kq_schema_id(f))) ? kq_qsb(f) : 256l + +//! plane bytes per row: kq formats per 256-superblock, q51 per 32-block (20 + 4), q8 n int8 + n/32 f32, mx4 n/2 nibbles + n/32 e8m0 +struct PlaneShape { + wq_row : int64 // the gemv's weight plane bytes per row + tq_row : int64 // the tile's weight plane bytes per row + ws_row : int64 // scale plane bytes per row +} + +def plane_shape(f : KqFmt; mx4 : bool; n : int64) : PlaneShape { + if (mx4) { + return PlaneShape(wq_row = n / 2l, tq_row = n / 2l, ws_row = n / 32l) + } + if (f == KqFmt.q8) { + return PlaneShape(wq_row = n, tq_row = n, ws_row = (n / 32l) * 4l) + } + if (f == KqFmt.q51) { + return PlaneShape(wq_row = (n / 32l) * 20l, tq_row = (n / 32l) * 20l, ws_row = (n / 32l) * 4l) + } + let nsb = n / 256l + return PlaneShape(wq_row = nsb * kq_qsb(f), tq_row = nsb * tile_kq_bytes_per_sb(f), ws_row = nsb * kq_ssb(f)) +} struct Planes { kq : array @@ -144,11 +170,10 @@ def sized(var a : array; n : int64) { a |> resize(n) } -def fill_planes(var p : Planes; f : KqFmt; n, d, ntok : int64) { - let nsb = n / 256l - p.kq |> sized(d * nsb * kq_qsb(f)) - p.tkq |> sized(d * nsb * tile_kq_bytes_per_sb(f)) - p.ks |> sized(d * nsb * kq_ssb(f)) +def fill_planes(var p : Planes; sh : PlaneShape; n, d, ntok : int64) { + p.kq |> sized(d * sh.wq_row) + p.tkq |> sized(d * sh.tq_row) + p.ks |> sized(d * sh.ws_row) p.xq |> sized(ntok * n) p.xs |> sized(ntok * n / 32l) p.xbs |> sized(ntok * n / 16l) @@ -178,20 +203,80 @@ def fill_planes(var p : Planes; f : KqFmt; n, d, ntok : int64) { } } -def emit_row(var rows : int&; f : KqFmt; tier, perm : string; n, d, ntok, best_us : int64; nsb : int64; plane_bytes : int64) { - let ns_per_sb = float(best_us) * 1000.0 / float(d * nsb * ntok) - let gbs = float(plane_bytes) / (float(best_us) * 1000.0) - let gflops = 2.0 * float(n) * float(d) * float(ntok) / (float(best_us) * 1000.0) - print("{f}\t{tier}\t{perm}\t{n}\t{d}\t{ntok}\t{best_us}\t{ns_per_sb}\t{gbs}\t{gflops}\n") +//! one timed row: the samples of every round; best and median are reported +struct RowStat { + perm : string + us : array +} + +def median_us(var us : array) : int64 { + var s := us + sort(s) + let m = length(s) / 2 + let r = length(s) % 2 == 1 ? s[m] : (s[m - 1] + s[m]) / 2l + delete s + return r +} + +def emit_row(var rows : int&; name, tier : string; var st : RowStat; n, d, ntok, nsb : int64; plane_bytes : int64) { + var best = 999999999l + for (u in st.us) { + best = min(best, u) + } + let med = median_us(st.us) + let ns_per_sb = float(best) * 1000.0 / float(d * nsb * ntok) + let gbs = float(plane_bytes) / (float(best) * 1000.0) + let gflops = 2.0 * float(n) * float(d) * float(ntok) / (float(best) * 1000.0) + let mode = get_env_variable("DAS_TUNE_MODE") + let perm = (st.perm == "reference" && mode != "tune" && mode != "test") ? "stamped" : st.perm // normal mode: the registry's one row IS the stamped winner + print("{name}\t{tier}\t{perm}\t{n}\t{d}\t{ntok}\t{best}\t{med}\t{ns_per_sb}\t{gbs}\t{gflops}\n") rows++ } -def bench_gemv(var rows : int&; cfg : BenchArgs; f : KqFmt; var p : Planes) { +def perm_wanted(spec, suffix : string) : bool { + if (empty(spec)) { + return true + } + for (tok in split(spec, ",")) { + let tk = strip(tok) + if (empty(tk)) { + continue + } + if (starts_with(tk, "=") ? suffix == slice(tk, 1) : find(suffix, tk) >= 0) { + return true + } + } + return false +} + +//! interleaved timing: round r calls every selected row once before round r + 1 starts +def time_rows(var stats : array; rounds : int; blk : block<(row : int) : void>) { + for (row in range(length(stats))) { + stats[row].us |> reserve(rounds) + invoke(blk, row) // warmup + codegen, unmeasured + } + for (_r in range(rounds)) { + for (row in range(length(stats))) { + let t0 = ref_time_ticks() + invoke(blk, row) + stats[row].us |> push(int64(get_time_usec(t0))) + } + } +} + +def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; var p : Planes; nsb : int64) { let n = int64(cfg.n) let d = int64(cfg.d) - let nsb = n / 256l let plane_bytes = long_length(p.kq) + long_length(p.ks) var vs <- gemv_variants(f) + var stats : array + var fns : array + for (v in vs) { + if (perm_wanted(cfg.perm, v._0)) { + stats |> emplace(RowStat(perm = v._0)) + fns |> push(v._1) + } + } unsafe { let kqp = addr(p.kq[0]) let ksp = addr(p.ks[0]) @@ -199,33 +284,43 @@ def bench_gemv(var rows : int&; cfg : BenchArgs; f : KqFmt; var p : Planes) { let xsp = addr(p.xs[0]) let xbsp = addr(p.xbs[0]) var yp = addr(p.y[0]) - for (v in vs) { - if (!perm_wanted(cfg.perm, v._0)) { - continue - } - invoke(v._1, yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) // warmup + codegen - var best_us = 999999999l - for (_r in range(cfg.rounds)) { - let t0 = ref_time_ticks() - invoke(v._1, yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) - best_us = min(best_us, int64(get_time_usec(t0))) - } - emit_row(rows, f, "gemv", v._0, n, d, 1l, best_us, nsb, plane_bytes) + time_rows(stats, cfg.rounds) $(row : int) { + invoke(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) } } + for (st in stats) { + emit_row(rows, name, "gemv", st, n, d, 1l, nsb, plane_bytes) + } + delete stats + delete fns delete vs } -def bench_tile(var rows : int&; cfg : BenchArgs; f : KqFmt; var p : Planes) { +def bench_tile_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; var p : Planes; nsb : int64; sh : PlaneShape) { let n = int64(cfg.n) let d = int64(cfg.d) let ntok = int64(cfg.ntok) - let nsb = n / 256l - let qsb = tile_kq_bytes_per_sb(f) - let ssb = kq_ssb(f) + let qrow = sh.tq_row + let srow = sh.ws_row let plane_bytes = long_length(p.tkq) + long_length(p.ks) - var mrs <- layout_mrs(f) + var mrs <- companion_ints(layout_variants(f)) var vs <- tile_variants(f) + var stats : array + var fns : array + var mrOf : array + for (v in vs) { + if (!perm_wanted(cfg.perm, v._0)) { + continue + } + let mr = mrs?[v._0] ?? (mrs?["reference"] ?? 0l) + if (mr <= 0l || d % mr != 0l) { + to_log(LOG_WARNING, "kq_kernel_bench: {name} tile '{v._0}' has no layout row (mr={mr}) - skipped\n") + continue + } + stats |> emplace(RowStat(perm = v._0)) + fns |> push(v._1) + mrOf |> push(mr) + } unsafe { let kqp = addr(p.tkq[0]) let ksp = addr(p.ks[0]) @@ -233,67 +328,168 @@ def bench_tile(var rows : int&; cfg : BenchArgs; f : KqFmt; var p : Planes) { let xsp = addr(p.xs[0]) let xbsp = addr(p.xbs[0]) var yp = addr(p.y[0]) + time_rows(stats, cfg.rounds) $(row : int) { + let mr = mrOf[row] + for (g in range64(d / mr)) { + var t0k = 0l + while (t0k < ntok) { + invoke(fns[row], yp, kqp + g * mr * qrow, ksp + g * mr * srow, xqp, xsp, xbsp, n, d, g, t0k) + t0k += TILE_TOKENS + } + } + } + } + for (st in stats) { + emit_row(rows, name, "tile", st, n, d, ntok, nsb, plane_bytes) + } + delete stats + delete fns + delete mrOf + delete vs + delete mrs +} + +//! the q8q8 grid's gemv rows (q8 or its mx4 companion) into stats + typed function lists +def q8_gemv_rows(cfg : BenchArgs; mx4 : bool; var stats : array; var fnsQ : array; var fnsM : array) { + if (mx4) { + var vs <- mx4q8_gemv_gen_variants() + for (v in vs) { + if (perm_wanted(cfg.perm, v._0)) { + stats |> emplace(RowStat(perm = v._0)) + fnsM |> push(v._1) + } + } + delete vs + } else { + var vs <- q8q8_gemv_gen_variants() for (v in vs) { - if (!perm_wanted(cfg.perm, v._0)) { - continue + if (perm_wanted(cfg.perm, v._0)) { + stats |> emplace(RowStat(perm = v._0)) + fnsQ |> push(v._1) } + } + delete vs + } +} + +//! the q8q8 grid's tile rows with each row's mr and tokens-per-call; skips rows whose mr the row count does not divide +def q8_tile_rows(cfg : BenchArgs; mx4 : bool; d : int64; var stats : array; var fnsQ : array; var fnsM : array; var mrOf, stepOf : array) { + var mrs <- companion_ints(q8q8_layout_gen_variants()) + var steps <- companion_ints(q8q8_tokstep_gen_variants()) + if (mx4) { + var vs <- mx4q8_tile_gen_variants() + for (v in vs) { let mr = mrs?[v._0] ?? (mrs?["reference"] ?? 0l) - if (mr <= 0l || d % mr != 0l) { - to_log(LOG_WARNING, "kq_kernel_bench: {f} tile '{v._0}' has no layout row (mr={mr}) - skipped\n") - continue + if (perm_wanted(cfg.perm, v._0) && mr > 0l && d % mr == 0l) { + stats |> emplace(RowStat(perm = v._0)) + fnsM |> push(v._1) + mrOf |> push(mr) + stepOf |> push(TILE_TOKENS) } - let ngroups = d / mr - var best_us = 999999999l - for (r in range(cfg.rounds + 1)) { // the first pass is warmup + codegen - let t0 = ref_time_ticks() - for (g in range64(ngroups)) { - var t0k = 0l - while (t0k < ntok) { - invoke(v._1, yp, kqp + g * mr * nsb * qsb, ksp + g * mr * nsb * ssb, xqp, xsp, xbsp, n, d, g, t0k) - t0k += TILE_TOKENS - } - } - if (r > 0) { - best_us = min(best_us, int64(get_time_usec(t0))) - } + } + delete vs + } else { + var vs <- q8q8_tile_gen_variants() + for (v in vs) { + let mr = mrs?[v._0] ?? (mrs?["reference"] ?? 0l) + if (perm_wanted(cfg.perm, v._0) && mr > 0l && d % mr == 0l) { + stats |> emplace(RowStat(perm = v._0)) + fnsQ |> push(v._1) + mrOf |> push(mr) + stepOf |> push(steps?[v._0] ?? (steps?["reference"] ?? TILE_TOKENS)) } - emit_row(rows, f, "tile", v._0, n, d, ntok, best_us, nsb, plane_bytes) } + delete vs } - delete vs delete mrs + delete steps } -def perm_wanted(spec, suffix : string) : bool { - if (empty(spec)) { - return true - } - for (tok in split(spec, ",")) { - let tk = strip(tok) - if (empty(tk)) { - continue +//! q8 (the q8q8 grid) and its mx4 companion: int8 / nibble weight planes with f32 / e8m0 scales +def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes; nsb : int64; sh : PlaneShape) { + let n = int64(cfg.n) + let d = int64(cfg.d) + let ntok = int64(cfg.ntok) + let name = mx4 ? MX4_NAME : "q8" + let plane_bytes = long_length(p.kq) + long_length(p.ks) + unsafe { + let kqp = addr(p.kq[0]) + let ksp = addr(p.ks[0]) + let xqp = addr(p.xq[0]) + let xsp = addr(p.xs[0]) + let xbsp = addr(p.xbs[0]) + var yp = addr(p.y[0]) + if (!cfg.no_gemv) { + var stats : array + var fnsQ : array + var fnsM : array + q8_gemv_rows(cfg, mx4, stats, fnsQ, fnsM) + time_rows(stats, cfg.rounds) $(row : int) { + if (mx4) { + invoke(fnsM[row], yp, kqp, ksp, xqp, xsp, n, 0l, d) + } else { + invoke(fnsQ[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, 0l, d) + } + } + for (st in stats) { + emit_row(rows, name, "gemv", st, n, d, 1l, nsb, plane_bytes) + } + delete stats + delete fnsQ + delete fnsM } - if (starts_with(tk, "=") ? suffix == slice(tk, 1) : find(suffix, tk) >= 0) { - return true + if (ntok > 0l) { + var stats : array + var fnsQ : array + var fnsM : array + var mrOf : array + var stepOf : array + q8_tile_rows(cfg, mx4, d, stats, fnsQ, fnsM, mrOf, stepOf) + time_rows(stats, cfg.rounds) $(row : int) { + let mr = mrOf[row] + for (g in range64(d / mr)) { + var t0k = 0l + while (t0k < ntok) { + if (mx4) { + invoke(fnsM[row], yp, kqp + g * mr * sh.tq_row, ksp + g * mr * sh.ws_row, xqp, xsp, xbsp, n, d, g, t0k) + } else { + invoke(fnsQ[row], yp, reinterpret(kqp + g * mr * sh.tq_row), reinterpret(ksp + g * mr * sh.ws_row), xqp, xsp, xbsp, n, d, g, t0k) + } + t0k += stepOf[row] + } + } + } + for (st in stats) { + emit_row(rows, name, "tile", st, n, d, ntok, nsb, plane_bytes) + } + delete stats + delete fnsQ + delete fnsM + delete mrOf + delete stepOf } } - return false } -def formats_of(spec : string) : array { - var fs : array +//! the requested formats as (name, KqFmt, is_mx4); "all" = every KqFmt row plus mx4 +def formats_of(spec : string) : array> { + var fs : array> if (spec == "all") { for (f in type) { - if (kq_sb(f)) { - fs |> push(f) - } + fs |> emplace(("{f}", f, false)) } + fs |> emplace((MX4_NAME, KqFmt.q8, true)) return <- fs } for (tok in split(spec, ",")) { let name = strip(tok) - if (!empty(name)) { - fs |> push(fmt_of_name(name)) + if (empty(name)) { + continue + } + if (name == MX4_NAME) { + fs |> emplace((MX4_NAME, KqFmt.q8, true)) + } else { + fs |> emplace((name, fmt_of_name(name), false)) } } return <- fs @@ -315,23 +511,29 @@ def main : int { var fs <- formats_of(cfg.fmt) if (!cfg.tsv) { print("kq_kernel_bench: n={cfg.n} d={cfg.d} ntok={cfg.ntok} rounds={cfg.rounds} tune_mode={get_env_variable("DAS_TUNE_MODE")}\n") - print("fmt\ttier\tperm\tn\td\tntok\tbest_us\tns_per_sb\tGB/s\tGFLOP/s\n") + print("fmt\ttier\tperm\tn\td\tntok\tbest_us\tmedian_us\tns_per_sb\tGB/s\tGFLOP/s\n") } var rows = 0 - for (f in fs) { + let nsb = int64(cfg.n) / 256l + for (fe in fs) { + let sh = plane_shape(fe._1, fe._2, int64(cfg.n)) var p : Planes - fill_planes(p, f, int64(cfg.n), int64(cfg.d), int64(max(cfg.ntok, 1))) - if (!cfg.no_gemv) { - bench_gemv(rows, cfg, f, p) - } - if (cfg.ntok > 0) { - bench_tile(rows, cfg, f, p) + fill_planes(p, sh, int64(cfg.n), int64(cfg.d), int64(max(cfg.ntok, 1))) + if (fe._2 || fe._1 == KqFmt.q8) { + bench_q8_family(rows, cfg, fe._2, p, nsb, sh) + } else { + if (!cfg.no_gemv) { + bench_gemv_kq(rows, cfg, fe._0, fe._1, p, nsb) + } + if (cfg.ntok > 0) { + bench_tile_kq(rows, cfg, fe._0, fe._1, p, nsb, sh) + } } delete p } delete fs if (rows == 0) { - to_log(LOG_ERROR, "kq_kernel_bench: no rows - nothing matched --fmt '{cfg.fmt}' or every arm was skipped\n") + to_log(LOG_ERROR, "kq_kernel_bench: no rows - nothing matched --fmt '{cfg.fmt}' / --perm '{cfg.perm}' or every arm was skipped\n") return 1 } return 0 diff --git a/modules/dasLLAMA/harness/README.md b/modules/dasLLAMA/harness/README.md index d18e7d5006..8617785fe2 100644 --- a/modules/dasLLAMA/harness/README.md +++ b/modules/dasLLAMA/harness/README.md @@ -8,6 +8,7 @@ instruments - each carries its purpose in its header comment. |---|---| | `parity.das` | dasLLAMA side of the token-for-token oracle check: prompt IDs in -> greedy generated IDs out (`GEN_IDS: ...`). Arch-agnostic - routes through `load_gguf` + `generate`. | | `parity.sh` | Runs the reference oracle and `parity.das` on the same prompt and diffs the two ID streams. | +| `kernel_ladder.sh` | The per-box kernel table: every format's stamped kernel (`benchmarks/matmul/kq_kernel_bench.das`, one thread, no model) against the reference exe's `test-backend-ops perf` at the same shape, decode and prefill rows, joined with the ratio. The reference binary carries the `GGML_BENCH_THREADS` define (a three-line edit of its `N_THREADS`; `backend_ops_shapes.patch` is the older, wider form). | | `oracle/simple_ids.cpp` | The reference oracle - links against a reference-engine build and prints greedy token IDs (CPU, no EOG break) rather than decoded text. | | `quant_eval.das` / `quant_eval_q4.das` | Q8 / Q4 reconstruction-error stats vs the fp32 weights. | | `bpe_test.das` | BPE tokenizer corpus gate against the in-repo `ggml-vocab-llama-bpe` fixtures (no model needed). | diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh new file mode 100644 index 0000000000..a996014882 --- /dev/null +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# kernel_ladder.sh - the per-box kernel table: every format's stamped kernel (kq_kernel_bench at +# one thread, normal tune mode = the winner this box ships) against the reference exe's +# test-backend-ops perf at the same shape, decode (n=1) and prefill (n=512) rows, joined with +# the ratio (reference us / ours us; >= 1.0 = ours is faster). No model, no jobque: the kernel +# and nothing else. The reference binary needs the GGML_BENCH_THREADS define (harness README). +# +# Usage: kernel_ladder.sh [fmt,fmt,...] (default: every format) +# KL_DASLANG the daslang binary (default: this tree's bin/daslang, or bin/Release/daslang.exe on +# Windows - never the global DASLANG, which names another tree's binary on dev boxes) +# LCPP_TBO test-backend-ops (default: $HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe) +# NTOK prefill tokens (default 512; 0 = decode rows only) +# ROUNDS interleaved rounds per row (default 5) +# Output: a TSV table on stdout - fmt tier perm ours_us ours_med_us ref_us ratio - plus the box line. +set -e -o pipefail # a failing daslang must not hide behind the join +ROOT=$(cd "$(dirname "$0")/../../.." && pwd) # the tree this script lives in, whatever the env says +DASLANG=${KL_DASLANG:-} +if [ -z "$DASLANG" ]; then + if [ -x "$ROOT/bin/Release/daslang.exe" ]; then DASLANG="$ROOT/bin/Release/daslang.exe"; else DASLANG="$ROOT/bin/daslang"; fi +fi +TBO=${LCPP_TBO:-$HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe} +FMTS=${1:-all} +NTOK=${NTOK:-512} +ROUNDS=${ROUNDS:-5} +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +# KqFmt name -> ggml type name (the reference's type_a) +ggml_type() { + case "$1" in + q8) echo q8_0;; k4) echo q4_K;; k5) echo q5_K;; k6) echo q6_K;; q40) echo q4_0;; q51) echo q5_1;; + iq4xs) echo iq4_xs;; k3) echo q3_K;; iq3s) echo iq3_s;; iq3xxs) echo iq3_xxs;; iq4nl) echo iq4_nl;; + k2) echo q2_K;; iq2s) echo iq2_s;; iq2xs) echo iq2_xs;; iq2xxs) echo iq2_xxs;; mx4) echo mxfp4;; + *) echo "";; + esac +} + +echo "# kernel ladder $(date +%F) box=$(hostname) daslang=$DASLANG ref=$TBO ntok=$NTOK rounds=$ROUNDS" >&2 + +# ---- ours: one process, every format, the stamped winner per format ---- +[ -x "$TBO" ] || { echo "kernel_ladder: no test-backend-ops at '$TBO' (set LCPP_TBO)" >&2; exit 1; } +set +e +DAS_TUNE_MODE=normal DAS_JOBQUE_THREADS=1 "$DASLANG" -jit "$ROOT/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das" \ + -- --fmt "$FMTS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv > "$WORK/ours.raw" 2> "$WORK/ours.err" +rc=$? +set -e +awk -F'\t' 'NF >= 11' "$WORK/ours.raw" > "$WORK/ours.tsv" || true # the TSV rows; engine and tune notes fall away +if [ $rc -ne 0 ] || [ ! -s "$WORK/ours.tsv" ]; then + echo "kernel_ladder: kq_kernel_bench failed (rc=$rc)" >&2 + tail -20 "$WORK/ours.err" >&2 + exit 1 +fi + +# ---- the reference: one perf run per type at one thread; n=1 and n=512 rows ---- +: > "$WORK/ref.tsv" +for f in $(cut -f1 "$WORK/ours.tsv" | sort -u); do + t=$(ggml_type "$f") + [ -n "$t" ] || { echo "kernel_ladder: no ggml type for '$f'" >&2; continue; } + GGML_BENCH_THREADS=1 "$TBO" perf -b CPU -o MUL_MAT -p "type_a=$t,type_b=f32,m=4096,n=(1|512)," 2>/dev/null \ + | sed 's/\x1b\[[0-9;]*m//g' | awk -v f="$f" ' + /MUL_MAT\(/ { + n = $0; sub(/.*,n=/, "", n); sub(/,.*/, "", n) + us = $0; sub(/.*runs - */, "", us); sub(/ us\/run.*/, "", us) + print f "\t" (n == 1 ? "gemv" : "tile") "\t" us + }' >> "$WORK/ref.tsv" +done + +# ---- join: ours (fmt tier perm n d ntok best med ...) x ref (fmt tier us) ---- +printf 'fmt\ttier\tperm\tours_us\tours_med_us\tref_us\tratio\n' +awk -F'\t' 'NR == FNR { ref[$1 "\t" $2] = $3; next } + { + key = $1 "\t" $2; r = (key in ref) ? ref[key] : "" + ratio = (r != "" && $7 > 0) ? sprintf("%.2f", r / $7) : "-" + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", $1, $2, $3, $7, $8, (r != "" ? r : "-"), ratio + }' "$WORK/ref.tsv" "$WORK/ours.tsv" diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index c5da565879..688d679128 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -50,6 +50,38 @@ proven fact per push; research before any kernel edit (the two memos below). | iq2xs | 11490 (50.1); sign column + u64 pair + column read 4831 (21.1) | 5386 | 0.47x -> 1.11x | - | - | | iq2xxs | 11061 (48.2); sign column + u64 pair 5121-5487 (noise band) | 5124 | 0.46x -> 0.93-1.00x | - | - | +The full ladder (`harness/kernel_ladder.sh`, 2026-09-01, zen2 one thread, the stamped seat per format, +best of 5 interleaved rounds; ratio = reference / ours, >= 1.00 = ours faster): + +| format | gemv ours us | gemv ref us | ratio | tile ours us | tile ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 3324 | 4186.70 | 1.26 | 698131 | 896060.00 | 1.28 | +| k4 | 2089 | 2539.11 | 1.22 | 493006 | 830068.50 | 1.68 | +| k5 | 3628 | 3286.40 | 0.91 | 512129 | 1344947.00 | 2.63 | +| k6 | 4203 | 3785.47 | 0.90 | 640544 | 1080212.00 | 1.69 | +| q40 | 1928 | 2918.57 | 1.51 | 380774 | 1004920.00 | 2.64 | +| q51 | 2816 | 4208.36 | 1.49 | 664024 | 1843932.00 | 2.78 | +| iq4xs | 2100 | 3396.72 | 1.62 | 721313 | 1682846.00 | 2.33 | +| k3 | 3317 | 2638.26 | 0.80 | 775632 | 1267418.00 | 1.63 | +| iq3s | 7671 | 10154.70 | 1.32 | 738713 | 5220515.00 | 7.07 | +| iq3xxs | 6622 | 6564.89 | 0.99 | 741832 | 3395150.00 | 4.58 | +| iq4nl | 2120 | 3134.77 | 1.48 | 599955 | 1076032.00 | 1.79 | +| k2 | 2146 | 2120.78 | 0.99 | 670297 | 783419.00 | 1.17 | +| iq2s | 5059 | 5083.57 | 1.00 | 790327 | 2545281.00 | 3.22 | +| iq2xs | 4554 | 5372.16 | 1.18 | 778490 | 2734115.00 | 3.51 | +| iq2xxs | 5143 | 5157.27 | 1.00 | 745463 | 2534210.00 | 3.40 | +| mx4 | 2572 | 3106.12 | 1.21 | 776635 | 1443088.00 | 1.86 | + +Every tile row is ahead (1.17x-7.07x). Decode tails: k3 0.80x, k6 0.90x, k5 0.91x; iq3xxs, k2, iq2s, +iq2xxs at 0.99-1.00 (inside the noise band). The 4-bit class (q40, q51, iq4xs, iq4nl, k4, q8, mx4) +sits at 1.2x-1.6x. + +Model level, zen2 16 threads, 1B vehicles, tg128 ours vs the clean-CPU llama-bench (before the pass): +IQ3_M (iq3s) 69.9 vs 56.5 = 1.24x (0.92x); IQ3_XXS-local 74.9 vs 74.3 = 1.01x (0.78x); IQ2_XS-local +93.8 vs 85.9 = 1.09x (0.70x); IQ2_XXS-local 88.6 vs 86.2 = 1.03x (0.70x); i1-IQ3_XXS (the mixed +iq2s/iq3xxs/iq3s vehicle) 71.9 +-4.2 vs 74.5 = 0.97x (0.76x). pp512 3.6x-5.0x throughout. Debug-jit +rows (`--for-debug-purposes`), 3 reps; the record-grade rows come with the released exe at arc end. + Reading: the five grid formats cost 48-51 ns per superblock regardless of what each decodes, against 8.6-8.8 for k4/k2. A flat cost independent of the format is a shared mechanism, not five decode problems. The model-level tg rows (0.70-0.92x) are this 2x kernel gap hidden behind memory @@ -85,7 +117,10 @@ bandwidth; on the 1B vehicles it shows, on a 27B it mostly does not - but the ke ## 4. Work queue (one item at a time; a row is done at >= 1.0x on rig 2's shape AND the vehicle) -CPU decode (gap 2): +CPU decode (gap 2) - the k-quant decode kernels are the tails the full ladder exposed: k3 0.80x, +k6 0.90x, k5 0.91x (next, k3 first); then the noise round on the 0.99-1.00 rows. + +CPU decode, the grid formats (done): 1. DONE-KILLED `gather="reg"` (measured 1.85x slower - see the ledger). 2. DONE all five: `sign="vec"` - the sign-byte column negates whole vectors; iq3s/iq2s load it off the plane, iq3xxs/iq2xs/iq2xxs synthesize it from the 7-bit codes (parity = the 8th bit), @@ -124,6 +159,12 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. One thread, m=4096 k=14336: iq3s 11581 -> 7732 us (1.34x of the reference's 10340), iq2s 12413 -> 7076 us (0.72x of 5074, was 0.41x). Every variant bit-exact in gen_tune_probe TEST mode. The five gather emitters collapsed into one gather over per-format decode functions (-110 lines). +- 2026-09-01: the lab completes - kq_kernel_bench carries all 16 formats (q8 + mx4 on the q8q8 + grid, q51 on its per-32 planes; the tile arm hands k5/k6 and the grid formats their byte-expanded + panel per kq_reads_packed_planes), rows interleaved round-robin with best + median; + harness/kernel_ladder.sh joins both sides into the per-box table (above). +- 2026-09-01: the 16-thread vehicle stamps (above) - every grid-format vehicle at or past the + reference on tg except the mixed i1-IQ3_XXS at 0.97x with a +-6% error bar. - 2026-09-01: the column dword read lands for iq3xxs and iq2xs only (measured per format, see the queue); zen2 one thread now: iq3s 7673, iq3xxs 6641, iq2s 5039-5174, iq2xs 4831, iq2xxs 5281-5487 us; 65 variants ok in TEST mode. From ddac6fbfe2e79a433624988e4aa008ae4a27a9f9 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 02:36:14 -0700 Subject: [PATCH 009/123] kernel parity, CPU k-quants: k3 and k2 flush their i16 chains once per sub-block; k3's decode re-loads its shared columns The per-16 chains of k3 (4 x 1778) and k2 (4 x 762) stay inside i16 for the whole sub-block, so the two mid-block widens go: k2 decode 2061 -> 1286-1423 us (reference 1951-2121). k3's decode shape re-loads the 24 qs/hmask column vectors its eight sub-blocks share (volatile loads, L1 hits) instead of CSE holding them live against 16 ymm registers and spilling: 2828-3198 -> 2445 us (reference 2638-2773). The pin is decode-only - on the tile it forbids the reordering the token reuse lives on (k3 tile 776 -> 1109 ms when tried). 65 variants ok in TEST mode. The k-quant research memo joins plans/. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 24 +- plans/kernel_parity_pass.md | 18 +- plans/kernel_parity_research_kquant.md | 445 ++++++++++++++++++ 3 files changed, 482 insertions(+), 5 deletions(-) create mode 100644 plans/kernel_parity_research_kquant.md diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 7a8eb6d1a9..b877e446ae 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -930,6 +930,16 @@ def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; na return LLVMBuildOr(b, w, sel, name) } +//! k3's decode re-loads the 24 shared qs/hmask column vectors where each sub-block uses them (L1 hits) +//! instead of CSE keeping them live against 16 ymm registers and spilling (3198 -> 2691 us). Volatile is +//! LLVM's "load here"; it also forbids reordering, so the tile shape (token reuse, ILP) keeps CSE. +def private pin_reload(var a, b, c, d : LLVMOpaqueValue?) { + LLVMSetVolatile(a, 1) + LLVMSetVolatile(b, 1) + LLVMSetVolatile(c, 1) + LLVMSetVolatile(d, 1) +} + // One 256-weight SUPERBLOCK, K-quant grp form (te.kq set): weight vectors unpacked // ONCE per (sub-block, dword-group) and dotted against every token — weight-stationary, // bit-exact vs per-token GEMVs (tokCount=1). kq v2 folds integer sub-scales/bsums against Q8_K-form activations. @@ -943,6 +953,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f let k2 = te.kq == 2 let k63 = k6 || k3 // k3 rides k6's scale row and fold; only the quant compose and the offset differ let per16 = k63 || k2 // per-16 scale granularity: split lo/hi accumulators, unsummed bs0/bs1 + let wholeSub = te.kq == 4 || te.kq == 40 || k3 || k2 // the i16 chains run the whole sub-block before one widen (k5: 4 madds, k6: 2) let koff = k3 ? 4 : 32 // the signed-lattice offset the bsum fold removes let q40 = te.kq == 40 let qsb = (te.kq == 4 || q40) ? 128 : (k3 ? 96 : (k2 ? 64 : (te.kqBytes ? 256 : (te.kq == 5 ? 160 : 192)))) @@ -1038,6 +1049,9 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var q3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((c0 + 4) * 4 * mr + qd * w8)), ""), "q3h{blk}_{j * rq + qd}") var h3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + j * 4) * mr + qd * w8)), ""), "h3l{blk}_{j * rq + qd}") var h3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + (j + 4) * 4) * mr + qd * w8)), ""), "h3h{blk}_{j * rq + qd}") + if (tokCount == 1 && rq == 1) { // the decode shape only: the tile's token reuse wants the columns kept live + pin_reload(q3l, q3h, h3l, h3h) + } var lo2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3l, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") var hi2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3h, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") var lob = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3l, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") @@ -1108,8 +1122,8 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } } - // i16 chain flushes at the format's overflow bound: k5 after 4 madds, k6 (and k3, conservatively) after 2; k4/q40 run the full 8 - if (madd16 && (j == 1 || j == 3) && te.kq != 4 && !q40) { + // i16 chain flushes at the format's overflow bound: k5 after 4 madds, k6 after 2 (2 x 16002); wholeSub formats run the sub-block + if (madd16 && (j == 1 || j == 3) && !wholeSub) { for (k in range(tokCount * rq)) { a0[k] = madd16_flush(te, a0[k], p16lo[k]) p16lo[k] = null @@ -1120,10 +1134,14 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } } - if (madd16 && (te.kq == 4 || q40)) { // one widen per (token, qd) for the whole sub-block + if (madd16 && wholeSub) { // one widen per (token, qd, chain) for the whole sub-block for (k in range(tokCount * rq)) { a0[k] = madd16_flush(te, a0[k], p16lo[k]) p16lo[k] = null + if (per16) { + a1[k] = madd16_flush(te, a1[k], p16hi[k]) + p16hi[k] = null + } } } // the integer fold — no float ops per block, except q40 (its per-32 f16 d admits no cross-block integer fold) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 688d679128..656005c922 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -117,8 +117,17 @@ bandwidth; on the 1B vehicles it shows, on a 27B it mostly does not - but the ke ## 4. Work queue (one item at a time; a row is done at >= 1.0x on rig 2's shape AND the vehicle) -CPU decode (gap 2) - the k-quant decode kernels are the tails the full ladder exposed: k3 0.80x, -k6 0.90x, k5 0.91x (next, k3 first); then the noise round on the 0.99-1.00 rows. +CPU decode (gap 2) - the k-quant decode kernels were the tails the full ladder exposed (k3 0.80x, +k6 0.90x, k5 0.91x): +1. DONE k3 + k2: one i16 flush per sub-block (the per-16 chains are bounded at 4 x 1778 / 4 x 762) - + k2 decode 2061 -> 1286-1423 us (reference 1951-2121); and k3's decode re-loads its 24 shared + qs/hmask columns per sub-block (volatile loads, decode shape only) instead of CSE keeping them live + and spilling - k3 2828-3198 -> 2445 us (reference 2638-2773). The pin hurt the tiles (k3 776 -> + 1109 ms) so it is decode-only; the principled form is the memo's D1 group-major reorder. +2. OPEN k6 0.87-0.90x, k5 0.87-0.91x: the pin gained nothing on k6; the memo (research_cpu_kquant.md) + sees no clean lever for k5 (D4) and offers a sub-block unroll knob (D5) for both. +3. Noise: the one-thread bench and the reference both drift ~5-8% run to run; interleaved rounds hold + ours steady, the reference is re-run per table. iq3xxs/iq2s/iq2xxs tie at 1.00; k2 now clear. CPU decode, the grid formats (done): 1. DONE-KILLED `gather="reg"` (measured 1.85x slower - see the ledger). @@ -159,6 +168,11 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. One thread, m=4096 k=14336: iq3s 11581 -> 7732 us (1.34x of the reference's 10340), iq2s 12413 -> 7076 us (0.72x of 5074, was 0.41x). Every variant bit-exact in gen_tune_probe TEST mode. The five gather emitters collapsed into one gather over per-format decode functions (-110 lines). +- 2026-09-01: research_cpu_kquant.md (llama.cpp q3_K/q5_K/q6_K vs ours): op counts are at parity; what + tracks the losses is the high-bit plane's column vectors staying live across the superblock after + CSE (k3 24 vectors, k6 16) against 16 ymm - the plane cannot be register-resident in our + 8-rows-per-vector layout the way it is in llama.cpp's row-major loop. Also: llama.cpp's q4_K has an + AVX2 repack GEMV that test-backend-ops perf does not run, so the k4 1.22x is against its fallback. - 2026-09-01: the lab completes - kq_kernel_bench carries all 16 formats (q8 + mx4 on the q8q8 grid, q51 on its per-32 planes; the tile arm hands k5/k6 and the grid formats their byte-expanded panel per kq_reads_packed_planes), rows interleaved round-robin with best + median; diff --git a/plans/kernel_parity_research_kquant.md b/plans/kernel_parity_research_kquant.md new file mode 100644 index 0000000000..9ea6e7eafd --- /dev/null +++ b/plans/kernel_parity_research_kquant.md @@ -0,0 +1,445 @@ +# Why our zen2 decode kernel loses on Q3_K / Q5_K / Q6_K and wins on Q4_K + +Read-only study. Reference tree `D:\Work\llama.cpp`, ours `D:\Work\daScript-iquant`. No file in +either tree was modified. + +Measured, zen2 AVX2, 1 thread, m=4096 k=14336, one token (us/call, ours vs llama.cpp): + +| fmt | ours | llama.cpp | ratio | weight+scale bytes / 256-weight row | +|---|---|---|---|---| +| q4_K | 2089 | 2539 | **1.22x ahead** | ours 148, llama 144 | +| q2_K | 2146 | 2121 | 0.99x | ours 84, llama 84 | +| q3_K | 3317 | 2638 | **0.80x** | ours 114, llama 110 | +| q5_K | 3628 | 3286 | **0.91x** | ours 180, llama 176 | +| q6_K | 4203 | 3785 | **0.90x** | ours 210, llama 210 | + +--- + +## 0. Which llama.cpp code path the reference numbers came from + +**The plain `ggml_vec_dot_*_q8_K` in `ggml/src/ggml-cpu/arch/x86/quants.c`, not the repack path.** + +- The repack ("extra buffer type") GEMV only fires when the weight tensor was allocated in + `ggml_backend_cpu_repack_buffer_type()` — `D:\Work\llama.cpp\ggml\src\ggml-cpu\repack.cpp:4774-4781`. +- `test-backend-ops` allocates every tensor with `ggml_backend_alloc_ctx_tensors(ctx, backend)`, i.e. + the backend's *default* buffer type — `D:\Work\llama.cpp\tests\test-backend-ops.cpp:1546`, + `:1555`, `:1805`. So the repack buffer type is never selected and the repack GEMV never runs. + +What repack support even exists on x86 matters for the endgame, though: + +- **q4_K has an AVX2 repack GEMV** — `repack.cpp:4599-4604` selects `q4_K_8x8_q8_K` on + `ggml_cpu_has_avx2()`; the kernel is `ggml_gemv_q4_K_8x8_q8_K`, + `ggml/src/ggml-cpu/arch/x86/repack.cpp:1464`, AVX2-guarded at `:1486`. +- **q2_K's repack is AVX512-only** — `repack.cpp:4627-4632` gates on `ggml_cpu_has_avx512()`; + the kernel `ggml_gemv_q2_K_8x8_q8_K` (`arch/x86/repack.cpp:1713`) is AVX2-guarded at `:1732` + but is never *selected* on a zen2 host. +- **q3_K, q5_K and q6_K have no x86 repack at all** — `repack.cpp:4644-4659` offers q5_K/q6_K + only under `ggml_cpu_has_neon()`; q3_K appears nowhere in the selector. + +So for q3/q5/q6 on zen2, the vec_dot we compared against is the only thing llama.cpp has, and our +comparison is fair. **For q4_K it is not the whole story**: a real llama.cpp run with a loaded model +would take `ggml_gemv_q4_K_8x8_q8_K` instead, which is a row-interleaved 8-column kernel much closer +in shape to ours. Our +22% on q4_K is against llama's *fallback*. + +**AVX512 does not change any of these four kernels.** Each function has exactly one +`#if defined __AVX2__ ... #elif defined __AVX__` pair and no AVX512 arm: q3_K `quants.c:1784`/`:1884`, +q4_K `:2057`/`:2122`, q5_K `:2235`/`:2314`, q6_K `:2439`/`:2510`. The VNNI helper that does exist +(`mul_sum_us8_pairs_float`, `quants.c:105-119`) is used only by the q4_0/q8_0-family dots; none of +the K-quant dots call it. On an AVX512 host llama.cpp runs the identical AVX2 code for these four. + +--- + +## 1. Per-format op budgets, side by side + +### Ground rules for the counts + +- llama.cpp: one 32-byte vector holds **32 consecutive weights of one row**; the activation vector + is a plain 32-byte `q8` load shared by that vector. +- Ours (seat `dot_maddubs_width256_mr8`, from + `D:\Work\daScript-iquant\modules\dasLLAMA\performance\defaults\x86-avx2.tune-defaults.json:35-38`, + giving `width=256`, `mr=8`, `dot=maddubs`; `rv = width/32 = 8`, `rq = mr/rv = 1` — + `dasllama_gemm_gen.das:1883-1884`): one 32-byte vector holds **4 bytes from each of 8 rows**; the + activation is one dword broadcast from memory (`bcast_dword`, `dasllama_gemm_gen.das:844-853`). + So one of our vectors also delivers 32 weight slots — the *unpack cost per vector* is the + apples-to-apples comparison. +- Our unit of emission is a **site** = one `(blk, j)` pair, `blk` in 0..7, `j` in 0..3, 32 sites per + superblock — `dasllama_gemm_gen.das:1002` (`for blk`), `:1027` (`for j`). Each site produces two + 32-byte vectors (`wlo`, `whi`). One superblock-group = 8 rows x 256 weights = 2048 slots. +- I count AVX2-level vector operations after the obvious InstCombine folds (in particular + `shl(and(lshr(X,a),m),b)` collapses to one shift plus one mask; LLVM does this before i8-vector + shift legalization, so our `(x >> a) & m << b` chains realize as 2 ops, not 4). Scalar prologue + arithmetic is not counted on either side. These are static counts, not measured uops. + +### Q4_K — the control we win + +**llama.cpp** (`quants.c:2057-2113`). Prologue: the 12-byte 6-bit scale/min packing is unpacked +with three scalar 32-bit words — `:2069-2074` (`memcpy(utmp,...,12)` then the kmask1/2/3 shuffle, +i.e. `get_scale_min_k4` done four sub-blocks at a time), widened once with `vpmovzxbw` at `:2079`. +The min term is folded **once per superblock**: bsums load `:2081`, `phaddw` `:2082`, +`pmaddwd` against the 8 mins `:2083`, one `fmadd` `:2084` — the whole eight-sub-block min dot is a +single `pmaddwd`, because the 8 mins of one row sit in one 128-bit register. + +Main loop `:2091-2110`, 4 iterations of 64 weights: + +| what | line | ops | +|---|---|---| +| 2 scale shuffles | 2093-2094 | 2 | +| 1 nibble load | 2096 | 1 | +| `q4l = and(bits, 0xF)` | 2097 | 1 | +| `q4h = and(srli(bits,4), 0xF)` | 2098 | 2 | +| 2 activation loads | 2100, 2104 | 2 | +| 2 `vpmaddubsw` | 2101, 2105 | 2 | +| 2 `vpmaddwd` (widen **and** scale in one op) | 2102, 2106 | 2 | +| 2 `vpaddd` | 2107, 2109 | 2 | + +15 per 64 weights x 4 = 60, plus ~15 prologue/epilogue vector ops -> **~75 ops per 256 weights of +one row**. Weight-plane loads issued: 4, distinct: 4 (1.00x). + +**Ours** (`dasllama_gemm_gen.das:1059-1062` for the unpack, `:1081-1108` for the dots): + +| what | line | ops/site | +|---|---|---| +| 1 nibble load (32B) | 1060 | 1 | +| `wlo = and(nv,15)` | 1061 | 1 | +| `whi = lshr(nv,4)` (vpsrlw+vpand on i8) | 1062 | 2 | +| 2 `vpbroadcastd` from memory | 1085-1086 / 844-853 | 2 | +| 2 `vpmaddubsw` + 2 `vpaddw` (i16 chain) | 1097-1101 / 872-878 | 4 | + +10 x 32 sites = 320. Per sub-block: one i16 flush (`vpmaddwd`+`vpaddd`, `:1123-1128`) = 2 x 8 = 16. +Per sub-block fold (`:1154-1160` scale/min loads, `:1181-1189` the integer fold): sc load + `vpmovzxbd` += 2, mn load + `vpmovzxbd` = 2, two scalar bsum loads + add + broadcast = 4, `vpmulld`+`vpaddd` for +`iacc` = 2, same for `bacc` = 2 -> 12 x 8 = 96. Superblock epilogue (`:1232-1250`) ~14. + +Total **~446 per 2048 slots = 55.8 ops per 256 weights of one row**. Weight-plane loads issued: 32, +distinct: 32 (1.00x). + +**Verdict: 55.8 vs 75 — we do 0.74x the work per weight.** Predicted 1.34x, measured 1.22x. + +Where the 19 ops come from: our vector covers 8 rows, so every *per-sub-block* cost (the scale +multiply, the min multiply, the accumulate, the flush) is paid once for 8 rows where llama pays it +once per row. Our fold is 12+2 = 14 ops per sub-block per 8 rows = 1.75 per row; llama's is +2 shuffles + 2 `vpmaddwd` + 2 `vpaddd` per 64 weights = 3 per 32 weights. That is the entire margin. +The *unpack* is a dead heat: 1 load + 3 ALU for two vectors, on both sides. + +### Q5_K — we lose 9% + +**llama.cpp** (`quants.c:2235-2308`). Scale unpack identical to q4_K (`:2252-2257`); min term folded +once per superblock via bsums (`:2261-2265`). **The whole 32-byte `qh` plane is loaded once, at +`:2270`, into one register, and a running `hmask` register walks it** (`:2271`, advanced by +`slli` at `:2287` and `:2292`). + +Main loop `:2278-2305`, 4 iterations of 64 weights: + +| what | line | ops | +|---|---|---| +| 2 scale shuffles | 2279-2280 | 2 | +| 1 nibble load | 2282 | 1 | +| `q5l_0 = and(bits, 0xF)` | 2284 | 1 | +| `q5h_0 = slli(srli(and(hbits,hmask),bit),4)` | 2285 | 3 | +| `q5_0 = q5l_0 + q5h_0` (`vpaddb`) | 2286 | 1 | +| `hmask <<= 1` | 2287 | 1 | +| `q5l_1 = and(srli(bits,4),0xF)` | 2289 | 2 | +| `q5h_1` | 2290 | 3 | +| `q5_1` + `hmask <<= 1` | 2291-2292 | 2 | +| 2 activation loads | 2294-2295 | 2 | +| 2 `vpmaddubsw` | 2297-2298 | 2 | +| 2 `vpmaddwd` (widen + scale) | 2300-2301 | 2 | +| 2 `vpaddd` | 2303 | 2 | + +22 x 4 = 88, plus ~16 prologue/epilogue -> **~104 ops per 256 weights of one row**. Weight-plane +loads issued: 4 nibble + 1 qh = 5, distinct 5 (1.00x). **Zero memory touches for the high bit inside +the loop.** + +**Ours** (`dasllama_gemm_gen.das:1064-1068`). k4's 4 ops for the nibbles, then per site: + +| what | line | ops/site | +|---|---|---| +| load 8 qh bytes (one per row) + broadcast to 4 lanes each | 1066 / `load_row_bytes_x4` 889-896 | 1 load + 2 (vpbroadcastq + vpshufb) | +| `or_bit_x10(wlo, hb, maskLo)` = and, cmpeqb, andn/and, or | 1067 / `or_bit_x10` 926-932 | 4 | +| `or_bit_x10(whi, hb, maskHi)` | 1068 | 4 | + +Site total: 2 loads + 13 ALU + 2 broadcasts + 4 madd/add = 21 -> 672. Flush at `j==1,3` +(`:1112`), one chain only since k5 is not per-16, 2 x 2 x 8 = 32. Fold 96. Epilogue ~14. + +Total **~814 per 2048 slots = 101.8 ops per 256 weights of one row**. Weight-plane loads issued: +32 nibble + 32 qh = 64, distinct 64 (1.00x). + +**Verdict: 101.8 vs 104 — op parity.** Predicted 1.02x, measured 0.91x. The 11% residual is not in +the op count; see section 3. + +Note the shape difference precisely: llama's high-bit deposit is **5 ops and 0 loads per 32 +weights**; ours is **4 ops per 32 slots plus a 3-op memory touch per 64 slots = 5.5 ops and 0.5 +loads per 32 slots**. Almost identical arithmetic, but ours re-enters memory 32 times per superblock +where llama enters it once. The reason is structural: llama's 32 lanes all want *the same* bit index, +so one scalar `bit` serves the whole vector; our 4 lanes of a row want bits t=0,1,2,3, so no uniform +shift exists and the bit must be selected with a per-lane constant mask (`maskLo`/`maskHi`, +`dasllama_gemm_gen.das:953-956`) that has to be re-applied to freshly loaded bytes each site. + +### Q6_K — we lose 10% + +**llama.cpp** (`quants.c:2439-2505`). Scales are native `int8[16]`, loaded raw at `:2455`. The `-32` +offset is removed **once per superblock** with a bsums trick: `q8sclsub = slli(madd(q8sums, +scales_16), 5)` at `:2457`, subtracted from `sumi` at `:2504`. So the dot itself is pure unsigned +`maddubs` on 0..63 values with no per-block correction at all. + +Main loop `:2463-2502`, 2 iterations of 128 weights. **One `qh` load feeds four sub-fields**: + +| what | line | ops | +|---|---|---| +| 2 `ql` loads + 1 `qh` load | 2464-2466 | 3 | +| `q4h_0 = slli(and(H,3),4)` | 2468 | 2 | +| `q4h_1 = slli(and(H,12),2)` | 2469 | 2 | +| `q4h_2 = and(H,48)` | 2470 | 1 | +| `q4h_3 = srli(and(H,0xC0),2)` | 2471 | 2 | +| `q4_0..q4_3 = or(and(ql,15) or and(srli(ql,4),15), q4h_*)` | 2473-2476 | 2+2+3+3 | +| 4 activation loads | 2478-2481 | 4 | +| 4 `vpmaddubsw` | 2483-2486 | 4 | +| 4 xmm scale shuffles | 2488-2491 | 4 | +| 4 `vpmovsxbw` + 4 `vpmaddwd` | 2494-2497 | 8 | +| 4 `vpaddd` | 2499-2500 | 4 | + +44 x 2 = 88, plus ~9 prologue/epilogue -> **~97 ops per 256 weights of one row**. Weight-plane loads +issued: 4 ql + 2 qh = 6, distinct 6 (1.00x). The mask constants are chosen so each 2-bit field is +isolated *and* positioned in one shift — no cleanup mask, because the mask already zeroed the bits +that would bleed across the 16-bit shift lane. + +**Ours** (`dasllama_gemm_gen.das:1069-1079`): + +| what | line | ops/site | +|---|---|---| +| 1 `ql` load + `and 15` + `lshr 4` | 1060-1062 | 1 load + 3 | +| **2 `qh` loads** (columns j and j+4) | 1074-1075 | 2 loads | +| `dlo = and(lshr(qhlo,g2),3)` | 1076 | 2 | +| `dhi = and(lshr(qhhi,g2),3)` | 1077 | 2 | +| `wlo |= dlo<<4`, `whi |= dhi<<4` | 1078-1079 | 4 | +| 2 broadcasts + 2 `vpmaddubsw` + 2 `vpaddw` | 1085-1101 | 6 | + +Site total 3 loads + 15 -> 18 x 32 = 576. Flush at `j==1,3` with **both** chains (k6 is per-16, +`:945`, `:1112-1120`) = 8 x 8 = 64. Fold per sub-block for the k63 arm (`:1139-1143` scale loads, +`:1188-1196` the two-scale integer fold): 2 int8 scale loads + 2 `vpmovsxbd` = 4, 2 scalar bsums + 2 +broadcasts = 4, `ia = sc*a0 + mn*a1` = 3, `ba = sc*bs0 + mn*bs1` = 3, 2 accumulate adds = 2 -> 16 x 8 += 128. Epilogue ~9. + +Total **~777 per 2048 slots = 97.1 ops per 256 weights of one row.** + +**Verdict: 97.1 vs 97 — exact op parity.** Predicted 1.00x, measured 0.90x. + +But: **weight-plane loads issued 96, distinct 48 — a 2.00x redundancy.** The offset at `:1071-1073` +is `hbase = 128*mr + (blk/4)*8*4*mr`, `hloOff = hbase + j*4*mr`, `hhiOff = hbase + (j+4)*4*mr` — it +depends on `blk/4` and `j` only, so **each of the 16 qh columns is re-loaded by all four `blk` values +in its group of four.** llama reads its qh plane exactly once. + +### Q3_K — we lose 20% + +**llama.cpp** (`quants.c:1784-1880`). The 12-byte 6-bit scale packing is decoded with four scalar +32-bit expressions into one xmm at `:1800-1805`, the `-32` bias applied with one `psubb` at `:1806`, +widened at `:1807`, split into two broadcast halves at `:1808-1810`. **The whole 32-byte `hmask` is +loaded once at `:1813`** and never re-read. + +The `-4` offset and the high bit are folded into a single subtraction: `q3h_j` is built with +`andnot(hbits, 1< **~108 ops per 256 weights of one row**. Weight-plane loads issued: +2 qs + 1 hmask = 3, distinct 3 (1.00x). + +**Ours** (`dasllama_gemm_gen.das:1033-1046`). We do **not** pay llama's double-maddubs: we compose an +unsigned 0..7 byte (`wlo = lo2 | (hbit<<2)`, `:1045-1046`) and remove the `-4` once per superblock +through the bsum fold (`koff = 4`, `:946`; applied at `:1238-1240`). One `maddubs` per vector. + +| what | line | ops/site | +|---|---|---| +| **4 loads**: `q3l` col `(blk/4)*8+j`, `q3h` col +4, `h3l` col `j`, `h3h` col `j+4` | 1037-1040 | 4 loads | +| `lo2 = and(lshr(q3l,sh),3)`, `hi2` | 1041-1042 | 4 | +| `lob = ((h3l>>blk)&1)<<2`, `hib` | 1043-1044 | 4 | +| 2 `vpor` | 1045-1046 | 2 | +| 2 broadcasts + 2 `vpmaddubsw` + 2 `vpaddw` | 1085-1101 | 6 | + +Site total 4 loads + 16 -> 20 x 32 = 640. Flush (per-16, two chains, at `j==1,3`) 64. Fold (k63 arm) +128. Epilogue ~9. Total **~841 per 2048 slots = 105 ops per 256 weights of one row.** + +**Verdict: 105 vs 108 — we are marginally *ahead* on op count.** Predicted 1.03x, measured 0.80x. + +But: **weight-plane loads issued 128, distinct 24 — a 5.33x redundancy**, the worst of the four. +Two independent causes, both visible in the offsets: +- `c0 = (blk/4)*8 + j` (`:1035`) -> each of the 16 qs columns is loaded by the 4 `blk` values in its + group (64 issued / 16 distinct). +- `h3l` offset `(64 + j*4)*mr` and `h3h` offset `(64 + (j+4)*4)*mr` (`:1039-1040`) **do not depend on + `blk` at all** -> each of the 8 hmask columns is loaded by all 8 `blk` values (64 issued / 8 + distinct). + +### The two tables together + +Ops per 256 weights of one row, and weight-plane load redundancy per superblock: + +| fmt | llama ops | our ops | our ops ratio | our loads issued/distinct | measured | +|---|---|---|---|---|---| +| q4_K | 75 | 55.8 | 0.74x (better) | 32 / 32 = **1.00x** | **1.22x** | +| q5_K | 104 | 101.8 | 0.98x | 64 / 64 = **1.00x** | 0.91x | +| q6_K | 97 | 97.1 | 1.00x | 96 / 48 = **2.00x** | 0.90x | +| q3_K | 108 | 105 | 0.97x | 128 / 24 = **5.33x** | 0.80x | + +Op count alone predicts +34% / +2% / 0% / +3%. The measured results are +22% / -9% / -10% / -20%. +**The op count explains q4_K and nothing else. The load-redundancy column is what tracks the losses.** + +--- + +## 2. The deltas, ranked by decode payoff + +Each entry says what it is: emitter change (IR only, no plane bytes move), repack-layout change +(plane bytes move -> `IMAGE_VERSION` bump, `dasllama_image.das:41`), or seat/knob. + +### D1 — q3_K: hoist the shared column loads out of the `blk` walk. **Emitter change.** +Payoff: largest single item on the board. 128 issued loads become 32 (the distinct count is 24), and +the CSE'd live set drops from 24 ymm values (16 qs columns + 8 hmask columns = 768 bytes, against 16 +architectural ymm registers) to about 4. + +Shape: keep the existing `(blk, j)` nest but split `blk` into a group of four +(`blk = 4*g + i`, g in 0..1, i in 0..3) and emit the loads at the `(g, j)` level. +For fixed `(g, j)` the four `blk` values share `q3l` (col `8g+j`), `q3h` (col `8g+j+4`), `h3l` +(col `j`) and `h3h` (col `j+4`) — see the offsets at `dasllama_gemm_gen.das:1037-1040` — and differ +only in the constant shifts `sh = 2*i` and `blk`. Live set inside the group: 4 loaded column vectors ++ 4 sub-blocks x 2 halves = 8 i16 chains, which fits. +No plane byte moves. No `IMAGE_VERSION` bump. No Vulkan/Metal impact (see D6 note). +Estimated: -96 loads/group (~-11% static ops) plus whatever spill traffic disappears; on the measured +0.80x this is the difference between "worst format" and "roughly at parity". + +### D2 — q6_K: the same hoist for the `qh` columns. **Emitter change.** +`hloOff`/`hhiOff` (`dasllama_gemm_gen.das:1072-1073`) depend on `blk/4` and `j`, not `blk`. Emitting +the two `qh` loads once per `(blk/4, j)` and looping the four `blk` values inside turns 64 qh loads +into 16. `ql` loads stay per-site (they are already 1.00x). Live set: 2 qh + 4 ql + 8 chains. +Estimated -48 loads/group (~-6% static ops) plus spill relief. + +### D3 — q3_K and q2_K: the i16 chain flushes twice as often as the bounds require. **Emitter change.** +`dasllama_gemm_gen.das:1112` flushes at `j==1` and `j==3` for every non-k4 format. The stated bound +(`madd16_acc`, `:869-871`) is one `vpmaddubsw` pair-sum <= `2*qmax*127`: +- k6 `2*63*127 = 16002` -> 2 madds = 32004, at the i16 limit. Correct as written. +- k5 `2*31*127 = 7874`, one chain carrying both halves -> 4 madds = 31496, at the limit. Correct. +- **k3 `2*7*127 = 1778`** -> 4 madds = 7112. A single flush per sub-block is safe (18 would be). +- **k2 `2*3*127 = 762`** -> 4 madds = 3048. Same. +The code comment at `:1110-1111` already says "k3, conservatively". Fix: make the flush predicate +`(j == 3)` for k3/k2. -4 ops per sub-block = -32/group, about -4% on k3 and -5% on k2. + +### D4 — q5_K: no clean lever exists; say so and stop looking. +Our k5 site is 2 loads + 13 ALU (`:1064-1068` plus the k4 nibble ops), llama's equivalent is +1 load-free deposit of 5 ops per 32 weights. Per weight the two are within 5% of each other and the +total op counts are 101.8 vs 104. The three candidate rewrites all fail: +- Rewriting `or_bit_x10` (`:926-932`) from `icmp-ne + select` to `vpand(vpcmpeqb(and,mask),16)` is the + same 4 ops. No win. +- **Repack-layout change**: pre-expanding the qh plane from 1 byte per (row, site) to 4 (so the bit is + already lane-local and the broadcast+shuffle at `:1066` disappears) saves about 4 ALU/site = + -128 ops/group (-16%), but grows the k5 weight plane from 160 to 256 bytes per superblock (+60%). + Our k5 currently moves 4096 x 56 x 180 B = 41.3 MB per call in 3628 us = 11.4 GB/s, against the ~16.3 GB/s our k4 demonstrates + is reachable on this core; the widened plane would be 58.7 MB, and even at 16.3 GB/s that is 3600 us -- no better than today. **Reject.** +- Splitting the dot into a nibble part plus a `16 *` high-bit part (llama's q3 trick applied to q5) + needs the same 4-op predicate to build the 0/1 vector, and adds a `maddubs`. **Reject.** +The honest read: q5_K's 9% is scheduling, not arithmetic — the per-site chain +`load(8B) -> vpbroadcastq -> vpshufb -> vpand -> vpcmpeqb -> vpandn -> vpor -> vpmaddubsw` is 8 deep +and repeats 32 times in one straight-line block. D5 is the lever that addresses it. + +### D5 — a sub-block unroll knob for the kq GEMV. **Emitter change + schema knob.** +`emit_block_kqv2` emits **all 32 sites of a superblock as straight-line IR** — `dasllama_gemm_gen.das:1002` +(`for blk in range(8)`) and `:1027` (`for j in range(4)`) are das-level compile-time loops, and the kq +GEMV driver hardcodes one superblock per emitted iteration (`SliceArgs(..., kstep = 1)`, +`dasllama_gemm_gen.das:2830`). That is 600-900 IR instructions in one basic block with 16 ymm +registers available. The generic `gkstep` knob documented at `:29-31` is not reachable from the kq +GEMV path. Adding an `sbstep` (sub-blocks per emitted loop iteration; 8 = today's behaviour, 2 and 4 +available) would give the tuner a handle on the live-set for every kq format at once, and is the only +proposal here that also helps q5_K. Costs a schema/perm-vector field; no plane bytes move. + +### D6 — q4_K scale-plane padding: identified, and not worth taking. +Our k4/k5 scale plane is 20 bytes per superblock (`K4_SSB`/`K5_SSB`, `dasllama_kqformat.das:94-96`): +2 d + 2 dmin + 8 decoded `sc` + 8 decoded `mn`, written by +`repack_k4_grp` (`dasllama_repack.das:218-224`) and `repack_k5_grp` (`:270-276`) — the 6-bit packing +is decoded once at load via `k4_sc_mn` (`dasllama_math_default.das:381-391`). llama keeps the packed +12 bytes and decodes in-register (`quants.c:2069-2074`), so its row is 16 bytes. That is +4 bytes per +superblock = **+2.8% weight-plane bytes**, and q4_K is the one format where we are bandwidth-bound +(4096 x 56 x 148 B = 34.0 MB per call in 2089 us = 16.3 GB/s, the highest rate any of the five reaches on this core). Reverting to +a packed plane would cost about 8 ALU ops per superblock-group against 32 bytes saved — roughly +break-even at our measured 2.65 bytes per op — for a repack-layout change and an `IMAGE_VERSION` +bump. Not worth it. The same reasoning says **keep** the k3 pre-decode: `K3_SSB = 18` +(`dasllama_kqformat.das:106`, "6-bit scale - 32, decoded at transcode") costs 4 bytes over llama's 14 +but buys us zero runtime scale decode, and k3 is compute-bound, not bandwidth-bound. + +### D7 — the seat/knob lane is empty for these formats. +`mr` must be a multiple of `rv = width/32 = 8` (`setup_tile_emit`, `dasllama_gemm_gen.das:1883-1884`), +so on AVX2 the only legal values are 8 and 16; 16 doubles the register pressure without changing any +redundancy factor (a wider column is still re-read once per `blk`). `kstep` is pinned to 1 for kq +(`:2830`). `nrsplit` is irrelevant at tokCount = 1. Zen2 has no VNNI, so `dot` is fixed at `maddubs`. +**No retune can move these numbers; every item above is code.** + +### Repack-layout / GPU-reader note +None of D1, D2, D3, D5 moves a plane byte, so none needs an `IMAGE_VERSION` bump. If a future item +does move the grp bytes, the blast radius is CPU-only: the grp repack runs at load and only when the +active backend asks for it (`dasllama_math.das:1182` `g_active_needs_repack`, recorded per model as +`kq_repacked`, `dasllama_load.das:2610`, `:2635`), while the GPU-resident tier reads the plane at the +**disk** stride (`t.k5q[sb0 * K5_QSB]`, `dasllama_gpu_resident.das:267-289`), not the grp layout. A +baked image does carry the repacked planes, so the `IMAGE_VERSION` bump is still required — but the +Vulkan and Metal readers are not on the hook. (Worth a one-line confirmation before anyone relies on +it.) + +--- + +## 3. Why q4_K wins and q5_K/q6_K lose in the same emitter — the hmask/qh suspect is CONFIRMED + +The suspect holds, and the correlation is perfect across all five formats: + +| fmt | high-bit plane? | our loss/win | +|---|---|---| +| q4_K | none | **+22%** | +| q2_K | none | -1% | +| q5_K | `qh`, 1 bit/weight | -9% | +| q6_K | `qh`, 2 bits/weight | -10% | +| q3_K | `hmask`, 1 bit/weight | -20% | + +**Every format with a separate high-bit plane loses; neither format without one does.** The mechanism +is not the arithmetic — section 1 shows we are at op parity or better on all three losers. It is +plane residency: + +- In llama.cpp's row-major layout, a 32-byte register holds the high bits for **32 consecutive + weights of one row**. Q3_K's entire 32-byte `hmask` covers the whole 256-weight superblock and is + loaded once (`quants.c:1813`). Q5_K's entire 32-byte `qh` likewise (`:2270`). Q6_K's `qh` is 64 + bytes, so it is loaded twice, one register per 128-weight iteration (`:2466`). **The high-bit plane + is register-resident for the whole loop, and extraction is pure ALU with a uniform shift, because + all 32 lanes want the same bit index.** +- In our `grp` layout a 32-byte register holds 4 bytes from each of 8 rows, so the high-bit plane + for the same amount of work is **mr = 8 times larger**: q5 256 bytes, q3 256 bytes (32 hmask x 8 + rows), q6 512 bytes — 8 to 16 ymm registers' worth, against 16 architectural registers on AVX2. + **It cannot be register-resident.** Every extraction therefore re-enters memory + (`dasllama_gemm_gen.das:1039-1040` for k3, `:1066` for k5, `:1074-1075` for k6), and because the + four lanes of a row want four different bit positions, the extraction needs a per-lane constant mask + (`maskLo`/`maskHi`, `:953-956`) rather than a uniform shift. +- Q4_K and Q2_K have no high-bit plane at all: their entire weight is inside the nibble/crumb byte + (`dasllama_gemm_gen.das:1060-1062` for k4, `:1051-1054` for k2). There is nothing to re-touch, our + loads are 1.00x, and the 8-row amortization of the fold shows up undiluted — hence +22% on k4. + +Q2_K's near-parity rather than a q4-sized win is the secondary effect and is also visible in the +code: k2 is a per-16-scale format (`per16 = k63 || k2`, `:945`), so it runs split lo/hi accumulators, +two i16 chains, two flushes per sub-block, and four `vpmulld` per sub-block in the fold +(`:1200-1207`) where k4 uses two — plus its plane is only 84 bytes per superblock, giving it about +1.08 bytes per vector op against k4's 2.65, i.e. it is compute-bound where k4 is bandwidth-bound. + +**Ranking of the three losers is set by the load-redundancy factor, not by the plane's bit width.** +q6_K carries twice as many high bits as q3_K but loses less, because its qh columns are re-read 4x +while q3_K's are re-read a weighted 5.33x overall (qs 4x *and* hmask 8x). That is why D1 (q3_K) is +the top item and D2 (q6_K) the second, and why both are pure loop-restructuring in the emitter with +no layout consequence. + +--- + +## 4. Two things to keep in view + +1. **The q4_K win is measured against llama.cpp's fallback.** `ggml_gemv_q4_K_8x8_q8_K` + (`arch/x86/repack.cpp:1464`, selected on AVX2 at `repack.cpp:4599-4604`) is what a real llama.cpp + model run uses on zen2, and it is an 8-column row-interleaved kernel structurally similar to ours. + Any claim of "1.22x ahead on q4_K" should be re-measured against that kernel before it leaves the + building. q3/q5/q6 have no such caveat — llama.cpp has no x86 repack for them. +2. **The prefill tiles are 1.6-2.6x ahead on the same formats**, which is consistent with everything + above: at tokCount = 4 the per-site unpack (including the redundant high-plane loads) is amortized + over four activation dwords instead of one, so the fraction of the loop spent on plane touching + drops by roughly 4x and the losers stop being losers. The decode kernel is the only place where + the unpack cost is paid at full weight. From e45ac5c8956328f1c9520e7a9754a9b95d2a9a27 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 02:37:16 -0700 Subject: [PATCH 010/123] kernel parity: the M1 ladder and the post-k3 zen2 rows in the plan Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- plans/kernel_parity_pass.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 656005c922..8056ea5928 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -72,7 +72,15 @@ best of 5 interleaved rounds; ratio = reference / ours, >= 1.00 = ours faster): | iq2xxs | 5143 | 5157.27 | 1.00 | 745463 | 2534210.00 | 3.40 | | mx4 | 2572 | 3106.12 | 1.21 | 776635 | 1443088.00 | 1.86 | -Every tile row is ahead (1.17x-7.07x). Decode tails: k3 0.80x, k6 0.90x, k5 0.91x; iq3xxs, k2, iq2s, +After the k3/k2 step (c720aea95): k3 decode 2445 us (1.08-1.13x), k2 1286-1423 (1.37-1.65x). + +The M1 ladder (Apple M1 Max, one thread, the sdot lattice at width 128, tip b88fc4100; ratio = +reference / ours): decode q8 1.46, k4 1.60, k5 1.06, k6 1.35, q40 1.96, q51 1.32, iq4xs 1.71, k3 +1.37, iq4nl 1.82, k2 1.42, mx4 1.50, iq2s 1.00, iq3s 0.93, iq3xxs 0.72, iq2xxs 0.57, iq2xs 0.51; +every tile 1.9x-15.3x ahead. On ARM the grid formats are the tails - the reference exe's NEON grid +kernels run iq2xs/iq2xxs at 3.3-3.4 ms against our 6.0-6.4 ms while the k-quants are 1.06x-1.60x. + +zen2 reading: every tile row is ahead (1.17x-7.07x). Decode tails were k3 0.80x, k6 0.90x, k5 0.91x; iq3xxs, k2, iq2s, iq2xxs at 0.99-1.00 (inside the noise band). The 4-bit class (q40, q51, iq4xs, iq4nl, k4, q8, mx4) sits at 1.2x-1.6x. @@ -150,6 +158,10 @@ CPU decode, the grid formats (done): 16-thread stamp on the vehicles (needs `--for-debug-purposes` on lcpp_bench, or the released exe). 8. retro audit of IQ4_XS/Q3_K per followup 60. +CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x - the grid +decode under the sdot lattice (width 128, two vectors per dword group). Untouched so far; a memo on +llama.cpp's NEON grid kernels vs our width-128 arms is the first step. + Vulkan pp (gap 1): 1. Budget split (measurement, no kernel edit) - is the GEMM the 30%? 2. `shAscales`-style scale hoist; 3. scale plane interleave; 4. N padding; 5+. the rest of the memo's list. From e0418cff5edcfd9123a772e63bc10b95b1740ff5 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 02:37:33 -0700 Subject: [PATCH 011/123] the k-quant memo through the Markdown ASCII gate Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- plans/kernel_parity_research_kquant.md | 84 +++++++++++++------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/plans/kernel_parity_research_kquant.md b/plans/kernel_parity_research_kquant.md index 9ea6e7eafd..a82e0ef17e 100644 --- a/plans/kernel_parity_research_kquant.md +++ b/plans/kernel_parity_research_kquant.md @@ -20,20 +20,20 @@ Measured, zen2 AVX2, 1 thread, m=4096 k=14336, one token (us/call, ours vs llama **The plain `ggml_vec_dot_*_q8_K` in `ggml/src/ggml-cpu/arch/x86/quants.c`, not the repack path.** - The repack ("extra buffer type") GEMV only fires when the weight tensor was allocated in - `ggml_backend_cpu_repack_buffer_type()` — `D:\Work\llama.cpp\ggml\src\ggml-cpu\repack.cpp:4774-4781`. + `ggml_backend_cpu_repack_buffer_type()` - `D:\Work\llama.cpp\ggml\src\ggml-cpu\repack.cpp:4774-4781`. - `test-backend-ops` allocates every tensor with `ggml_backend_alloc_ctx_tensors(ctx, backend)`, i.e. - the backend's *default* buffer type — `D:\Work\llama.cpp\tests\test-backend-ops.cpp:1546`, + the backend's *default* buffer type - `D:\Work\llama.cpp\tests\test-backend-ops.cpp:1546`, `:1555`, `:1805`. So the repack buffer type is never selected and the repack GEMV never runs. What repack support even exists on x86 matters for the endgame, though: -- **q4_K has an AVX2 repack GEMV** — `repack.cpp:4599-4604` selects `q4_K_8x8_q8_K` on +- **q4_K has an AVX2 repack GEMV** - `repack.cpp:4599-4604` selects `q4_K_8x8_q8_K` on `ggml_cpu_has_avx2()`; the kernel is `ggml_gemv_q4_K_8x8_q8_K`, `ggml/src/ggml-cpu/arch/x86/repack.cpp:1464`, AVX2-guarded at `:1486`. -- **q2_K's repack is AVX512-only** — `repack.cpp:4627-4632` gates on `ggml_cpu_has_avx512()`; +- **q2_K's repack is AVX512-only** - `repack.cpp:4627-4632` gates on `ggml_cpu_has_avx512()`; the kernel `ggml_gemv_q2_K_8x8_q8_K` (`arch/x86/repack.cpp:1713`) is AVX2-guarded at `:1732` but is never *selected* on a zen2 host. -- **q3_K, q5_K and q6_K have no x86 repack at all** — `repack.cpp:4644-4659` offers q5_K/q6_K +- **q3_K, q5_K and q6_K have no x86 repack at all** - `repack.cpp:4644-4659` offers q5_K/q6_K only under `ggml_cpu_has_neon()`; q3_K appears nowhere in the selector. So for q3/q5/q6 on zen2, the vec_dot we compared against is the only thing llama.cpp has, and our @@ -57,26 +57,26 @@ the K-quant dots call it. On an AVX512 host llama.cpp runs the identical AVX2 co is a plain 32-byte `q8` load shared by that vector. - Ours (seat `dot_maddubs_width256_mr8`, from `D:\Work\daScript-iquant\modules\dasLLAMA\performance\defaults\x86-avx2.tune-defaults.json:35-38`, - giving `width=256`, `mr=8`, `dot=maddubs`; `rv = width/32 = 8`, `rq = mr/rv = 1` — + giving `width=256`, `mr=8`, `dot=maddubs`; `rv = width/32 = 8`, `rq = mr/rv = 1` - `dasllama_gemm_gen.das:1883-1884`): one 32-byte vector holds **4 bytes from each of 8 rows**; the activation is one dword broadcast from memory (`bcast_dword`, `dasllama_gemm_gen.das:844-853`). - So one of our vectors also delivers 32 weight slots — the *unpack cost per vector* is the + So one of our vectors also delivers 32 weight slots - the *unpack cost per vector* is the apples-to-apples comparison. - Our unit of emission is a **site** = one `(blk, j)` pair, `blk` in 0..7, `j` in 0..3, 32 sites per - superblock — `dasllama_gemm_gen.das:1002` (`for blk`), `:1027` (`for j`). Each site produces two + superblock - `dasllama_gemm_gen.das:1002` (`for blk`), `:1027` (`for j`). Each site produces two 32-byte vectors (`wlo`, `whi`). One superblock-group = 8 rows x 256 weights = 2048 slots. - I count AVX2-level vector operations after the obvious InstCombine folds (in particular `shl(and(lshr(X,a),m),b)` collapses to one shift plus one mask; LLVM does this before i8-vector shift legalization, so our `(x >> a) & m << b` chains realize as 2 ops, not 4). Scalar prologue arithmetic is not counted on either side. These are static counts, not measured uops. -### Q4_K — the control we win +### Q4_K - the control we win **llama.cpp** (`quants.c:2057-2113`). Prologue: the 12-byte 6-bit scale/min packing is unpacked -with three scalar 32-bit words — `:2069-2074` (`memcpy(utmp,...,12)` then the kmask1/2/3 shuffle, +with three scalar 32-bit words - `:2069-2074` (`memcpy(utmp,...,12)` then the kmask1/2/3 shuffle, i.e. `get_scale_min_k4` done four sub-blocks at a time), widened once with `vpmovzxbw` at `:2079`. The min term is folded **once per superblock**: bsums load `:2081`, `phaddw` `:2082`, -`pmaddwd` against the 8 mins `:2083`, one `fmadd` `:2084` — the whole eight-sub-block min dot is a +`pmaddwd` against the 8 mins `:2083`, one `fmadd` `:2084` - the whole eight-sub-block min dot is a single `pmaddwd`, because the 8 mins of one row sit in one 128-bit register. Main loop `:2091-2110`, 4 iterations of 64 weights: @@ -113,7 +113,7 @@ Per sub-block fold (`:1154-1160` scale/min loads, `:1181-1189` the integer fold) Total **~446 per 2048 slots = 55.8 ops per 256 weights of one row**. Weight-plane loads issued: 32, distinct: 32 (1.00x). -**Verdict: 55.8 vs 75 — we do 0.74x the work per weight.** Predicted 1.34x, measured 1.22x. +**Verdict: 55.8 vs 75 - we do 0.74x the work per weight.** Predicted 1.34x, measured 1.22x. Where the 19 ops come from: our vector covers 8 rows, so every *per-sub-block* cost (the scale multiply, the min multiply, the accumulate, the flush) is paid once for 8 rows where llama pays it @@ -121,7 +121,7 @@ once per row. Our fold is 12+2 = 14 ops per sub-block per 8 rows = 1.75 per row; 2 shuffles + 2 `vpmaddwd` + 2 `vpaddd` per 64 weights = 3 per 32 weights. That is the entire margin. The *unpack* is a dead heat: 1 load + 3 ALU for two vectors, on both sides. -### Q5_K — we lose 9% +### Q5_K - we lose 9% **llama.cpp** (`quants.c:2235-2308`). Scale unpack identical to q4_K (`:2252-2257`); min term folded once per superblock via bsums (`:2261-2265`). **The whole 32-byte `qh` plane is loaded once, at @@ -164,7 +164,7 @@ Site total: 2 loads + 13 ALU + 2 broadcasts + 4 madd/add = 21 -> 672. Flush at ` Total **~814 per 2048 slots = 101.8 ops per 256 weights of one row**. Weight-plane loads issued: 32 nibble + 32 qh = 64, distinct 64 (1.00x). -**Verdict: 101.8 vs 104 — op parity.** Predicted 1.02x, measured 0.91x. The 11% residual is not in +**Verdict: 101.8 vs 104 - op parity.** Predicted 1.02x, measured 0.91x. The 11% residual is not in the op count; see section 3. Note the shape difference precisely: llama's high-bit deposit is **5 ops and 0 loads per 32 @@ -175,7 +175,7 @@ so one scalar `bit` serves the whole vector; our 4 lanes of a row want bits t=0, shift exists and the bit must be selected with a per-lane constant mask (`maskLo`/`maskHi`, `dasllama_gemm_gen.das:953-956`) that has to be re-applied to freshly loaded bytes each site. -### Q6_K — we lose 10% +### Q6_K - we lose 10% **llama.cpp** (`quants.c:2439-2505`). Scales are native `int8[16]`, loaded raw at `:2455`. The `-32` offset is removed **once per superblock** with a bsums trick: `q8sclsub = slli(madd(q8sums, @@ -200,7 +200,7 @@ Main loop `:2463-2502`, 2 iterations of 128 weights. **One `qh` load feeds four 44 x 2 = 88, plus ~9 prologue/epilogue -> **~97 ops per 256 weights of one row**. Weight-plane loads issued: 4 ql + 2 qh = 6, distinct 6 (1.00x). The mask constants are chosen so each 2-bit field is -isolated *and* positioned in one shift — no cleanup mask, because the mask already zeroed the bits +isolated *and* positioned in one shift - no cleanup mask, because the mask already zeroed the bits that would bleed across the 16-bit shift lane. **Ours** (`dasllama_gemm_gen.das:1069-1079`): @@ -222,14 +222,14 @@ broadcasts = 4, `ia = sc*a0 + mn*a1` = 3, `ba = sc*bs0 + mn*bs1` = 3, 2 accumula Total **~777 per 2048 slots = 97.1 ops per 256 weights of one row.** -**Verdict: 97.1 vs 97 — exact op parity.** Predicted 1.00x, measured 0.90x. +**Verdict: 97.1 vs 97 - exact op parity.** Predicted 1.00x, measured 0.90x. -But: **weight-plane loads issued 96, distinct 48 — a 2.00x redundancy.** The offset at `:1071-1073` -is `hbase = 128*mr + (blk/4)*8*4*mr`, `hloOff = hbase + j*4*mr`, `hhiOff = hbase + (j+4)*4*mr` — it +But: **weight-plane loads issued 96, distinct 48 - a 2.00x redundancy.** The offset at `:1071-1073` +is `hbase = 128*mr + (blk/4)*8*4*mr`, `hloOff = hbase + j*4*mr`, `hhiOff = hbase + (j+4)*4*mr` - it depends on `blk/4` and `j` only, so **each of the 16 qh columns is re-loaded by all four `blk` values in its group of four.** llama reads its qh plane exactly once. -### Q3_K — we lose 20% +### Q3_K - we lose 20% **llama.cpp** (`quants.c:1784-1880`). The 12-byte 6-bit scale packing is decoded with four scalar 32-bit expressions into one xmm at `:1800-1805`, the `-32` bias applied with one `psubb` at `:1806`, @@ -262,9 +262,9 @@ through the bsum fold (`koff = 4`, `:946`; applied at `:1238-1240`). One `maddub Site total 4 loads + 16 -> 20 x 32 = 640. Flush (per-16, two chains, at `j==1,3`) 64. Fold (k63 arm) 128. Epilogue ~9. Total **~841 per 2048 slots = 105 ops per 256 weights of one row.** -**Verdict: 105 vs 108 — we are marginally *ahead* on op count.** Predicted 1.03x, measured 0.80x. +**Verdict: 105 vs 108 - we are marginally *ahead* on op count.** Predicted 1.03x, measured 0.80x. -But: **weight-plane loads issued 128, distinct 24 — a 5.33x redundancy**, the worst of the four. +But: **weight-plane loads issued 128, distinct 24 - a 5.33x redundancy**, the worst of the four. Two independent causes, both visible in the offsets: - `c0 = (blk/4)*8 + j` (`:1035`) -> each of the 16 qs columns is loaded by the 4 `blk` values in its group (64 issued / 16 distinct). @@ -293,7 +293,7 @@ Op count alone predicts +34% / +2% / 0% / +3%. The measured results are +22% / - Each entry says what it is: emitter change (IR only, no plane bytes move), repack-layout change (plane bytes move -> `IMAGE_VERSION` bump, `dasllama_image.das:41`), or seat/knob. -### D1 — q3_K: hoist the shared column loads out of the `blk` walk. **Emitter change.** +### D1 - q3_K: hoist the shared column loads out of the `blk` walk. **Emitter change.** Payoff: largest single item on the board. 128 issued loads become 32 (the distinct count is 24), and the CSE'd live set drops from 24 ymm values (16 qs columns + 8 hmask columns = 768 bytes, against 16 architectural ymm registers) to about 4. @@ -301,20 +301,20 @@ architectural ymm registers) to about 4. Shape: keep the existing `(blk, j)` nest but split `blk` into a group of four (`blk = 4*g + i`, g in 0..1, i in 0..3) and emit the loads at the `(g, j)` level. For fixed `(g, j)` the four `blk` values share `q3l` (col `8g+j`), `q3h` (col `8g+j+4`), `h3l` -(col `j`) and `h3h` (col `j+4`) — see the offsets at `dasllama_gemm_gen.das:1037-1040` — and differ +(col `j`) and `h3h` (col `j+4`) - see the offsets at `dasllama_gemm_gen.das:1037-1040` - and differ only in the constant shifts `sh = 2*i` and `blk`. Live set inside the group: 4 loaded column vectors + 4 sub-blocks x 2 halves = 8 i16 chains, which fits. No plane byte moves. No `IMAGE_VERSION` bump. No Vulkan/Metal impact (see D6 note). Estimated: -96 loads/group (~-11% static ops) plus whatever spill traffic disappears; on the measured 0.80x this is the difference between "worst format" and "roughly at parity". -### D2 — q6_K: the same hoist for the `qh` columns. **Emitter change.** +### D2 - q6_K: the same hoist for the `qh` columns. **Emitter change.** `hloOff`/`hhiOff` (`dasllama_gemm_gen.das:1072-1073`) depend on `blk/4` and `j`, not `blk`. Emitting the two `qh` loads once per `(blk/4, j)` and looping the four `blk` values inside turns 64 qh loads into 16. `ql` loads stay per-site (they are already 1.00x). Live set: 2 qh + 4 ql + 8 chains. Estimated -48 loads/group (~-6% static ops) plus spill relief. -### D3 — q3_K and q2_K: the i16 chain flushes twice as often as the bounds require. **Emitter change.** +### D3 - q3_K and q2_K: the i16 chain flushes twice as often as the bounds require. **Emitter change.** `dasllama_gemm_gen.das:1112` flushes at `j==1` and `j==3` for every non-k4 format. The stated bound (`madd16_acc`, `:869-871`) is one `vpmaddubsw` pair-sum <= `2*qmax*127`: - k6 `2*63*127 = 16002` -> 2 madds = 32004, at the i16 limit. Correct as written. @@ -324,7 +324,7 @@ Estimated -48 loads/group (~-6% static ops) plus spill relief. The code comment at `:1110-1111` already says "k3, conservatively". Fix: make the flush predicate `(j == 3)` for k3/k2. -4 ops per sub-block = -32/group, about -4% on k3 and -5% on k2. -### D4 — q5_K: no clean lever exists; say so and stop looking. +### D4 - q5_K: no clean lever exists; say so and stop looking. Our k5 site is 2 loads + 13 ALU (`:1064-1068` plus the k4 nibble ops), llama's equivalent is 1 load-free deposit of 5 ops per 32 weights. Per weight the two are within 5% of each other and the total op counts are 101.8 vs 104. The three candidate rewrites all fail: @@ -337,12 +337,12 @@ total op counts are 101.8 vs 104. The three candidate rewrites all fail: is reachable on this core; the widened plane would be 58.7 MB, and even at 16.3 GB/s that is 3600 us -- no better than today. **Reject.** - Splitting the dot into a nibble part plus a `16 *` high-bit part (llama's q3 trick applied to q5) needs the same 4-op predicate to build the 0/1 vector, and adds a `maddubs`. **Reject.** -The honest read: q5_K's 9% is scheduling, not arithmetic — the per-site chain +The honest read: q5_K's 9% is scheduling, not arithmetic - the per-site chain `load(8B) -> vpbroadcastq -> vpshufb -> vpand -> vpcmpeqb -> vpandn -> vpor -> vpmaddubsw` is 8 deep and repeats 32 times in one straight-line block. D5 is the lever that addresses it. -### D5 — a sub-block unroll knob for the kq GEMV. **Emitter change + schema knob.** -`emit_block_kqv2` emits **all 32 sites of a superblock as straight-line IR** — `dasllama_gemm_gen.das:1002` +### D5 - a sub-block unroll knob for the kq GEMV. **Emitter change + schema knob.** +`emit_block_kqv2` emits **all 32 sites of a superblock as straight-line IR** - `dasllama_gemm_gen.das:1002` (`for blk in range(8)`) and `:1027` (`for j in range(4)`) are das-level compile-time loops, and the kq GEMV driver hardcodes one superblock per emitted iteration (`SliceArgs(..., kstep = 1)`, `dasllama_gemm_gen.das:2830`). That is 600-900 IR instructions in one basic block with 16 ymm @@ -351,21 +351,21 @@ GEMV path. Adding an `sbstep` (sub-blocks per emitted loop iteration; 8 = today' available) would give the tuner a handle on the live-set for every kq format at once, and is the only proposal here that also helps q5_K. Costs a schema/perm-vector field; no plane bytes move. -### D6 — q4_K scale-plane padding: identified, and not worth taking. +### D6 - q4_K scale-plane padding: identified, and not worth taking. Our k4/k5 scale plane is 20 bytes per superblock (`K4_SSB`/`K5_SSB`, `dasllama_kqformat.das:94-96`): 2 d + 2 dmin + 8 decoded `sc` + 8 decoded `mn`, written by -`repack_k4_grp` (`dasllama_repack.das:218-224`) and `repack_k5_grp` (`:270-276`) — the 6-bit packing +`repack_k4_grp` (`dasllama_repack.das:218-224`) and `repack_k5_grp` (`:270-276`) - the 6-bit packing is decoded once at load via `k4_sc_mn` (`dasllama_math_default.das:381-391`). llama keeps the packed 12 bytes and decodes in-register (`quants.c:2069-2074`), so its row is 16 bytes. That is +4 bytes per superblock = **+2.8% weight-plane bytes**, and q4_K is the one format where we are bandwidth-bound (4096 x 56 x 148 B = 34.0 MB per call in 2089 us = 16.3 GB/s, the highest rate any of the five reaches on this core). Reverting to -a packed plane would cost about 8 ALU ops per superblock-group against 32 bytes saved — roughly -break-even at our measured 2.65 bytes per op — for a repack-layout change and an `IMAGE_VERSION` +a packed plane would cost about 8 ALU ops per superblock-group against 32 bytes saved - roughly +break-even at our measured 2.65 bytes per op - for a repack-layout change and an `IMAGE_VERSION` bump. Not worth it. The same reasoning says **keep** the k3 pre-decode: `K3_SSB = 18` (`dasllama_kqformat.das:106`, "6-bit scale - 32, decoded at transcode") costs 4 bytes over llama's 14 but buys us zero runtime scale decode, and k3 is compute-bound, not bandwidth-bound. -### D7 — the seat/knob lane is empty for these formats. +### D7 - the seat/knob lane is empty for these formats. `mr` must be a multiple of `rv = width/32 = 8` (`setup_tile_emit`, `dasllama_gemm_gen.das:1883-1884`), so on AVX2 the only legal values are 8 and 16; 16 doubles the register pressure without changing any redundancy factor (a wider column is still re-read once per `blk`). `kstep` is pinned to 1 for kq @@ -378,13 +378,13 @@ does move the grp bytes, the blast radius is CPU-only: the grp repack runs at lo active backend asks for it (`dasllama_math.das:1182` `g_active_needs_repack`, recorded per model as `kq_repacked`, `dasllama_load.das:2610`, `:2635`), while the GPU-resident tier reads the plane at the **disk** stride (`t.k5q[sb0 * K5_QSB]`, `dasllama_gpu_resident.das:267-289`), not the grp layout. A -baked image does carry the repacked planes, so the `IMAGE_VERSION` bump is still required — but the +baked image does carry the repacked planes, so the `IMAGE_VERSION` bump is still required - but the Vulkan and Metal readers are not on the hook. (Worth a one-line confirmation before anyone relies on it.) --- -## 3. Why q4_K wins and q5_K/q6_K lose in the same emitter — the hmask/qh suspect is CONFIRMED +## 3. Why q4_K wins and q5_K/q6_K lose in the same emitter - the hmask/qh suspect is CONFIRMED The suspect holds, and the correlation is perfect across all five formats: @@ -397,7 +397,7 @@ The suspect holds, and the correlation is perfect across all five formats: | q3_K | `hmask`, 1 bit/weight | -20% | **Every format with a separate high-bit plane loses; neither format without one does.** The mechanism -is not the arithmetic — section 1 shows we are at op parity or better on all three losers. It is +is not the arithmetic - section 1 shows we are at op parity or better on all three losers. It is plane residency: - In llama.cpp's row-major layout, a 32-byte register holds the high bits for **32 consecutive @@ -408,19 +408,19 @@ plane residency: all 32 lanes want the same bit index.** - In our `grp` layout a 32-byte register holds 4 bytes from each of 8 rows, so the high-bit plane for the same amount of work is **mr = 8 times larger**: q5 256 bytes, q3 256 bytes (32 hmask x 8 - rows), q6 512 bytes — 8 to 16 ymm registers' worth, against 16 architectural registers on AVX2. + rows), q6 512 bytes - 8 to 16 ymm registers' worth, against 16 architectural registers on AVX2. **It cannot be register-resident.** Every extraction therefore re-enters memory (`dasllama_gemm_gen.das:1039-1040` for k3, `:1066` for k5, `:1074-1075` for k6), and because the four lanes of a row want four different bit positions, the extraction needs a per-lane constant mask (`maskLo`/`maskHi`, `:953-956`) rather than a uniform shift. - Q4_K and Q2_K have no high-bit plane at all: their entire weight is inside the nibble/crumb byte (`dasllama_gemm_gen.das:1060-1062` for k4, `:1051-1054` for k2). There is nothing to re-touch, our - loads are 1.00x, and the 8-row amortization of the fold shows up undiluted — hence +22% on k4. + loads are 1.00x, and the 8-row amortization of the fold shows up undiluted - hence +22% on k4. Q2_K's near-parity rather than a q4-sized win is the secondary effect and is also visible in the code: k2 is a per-16-scale format (`per16 = k63 || k2`, `:945`), so it runs split lo/hi accumulators, two i16 chains, two flushes per sub-block, and four `vpmulld` per sub-block in the fold -(`:1200-1207`) where k4 uses two — plus its plane is only 84 bytes per superblock, giving it about +(`:1200-1207`) where k4 uses two - plus its plane is only 84 bytes per superblock, giving it about 1.08 bytes per vector op against k4's 2.65, i.e. it is compute-bound where k4 is bandwidth-bound. **Ranking of the three losers is set by the load-redundancy factor, not by the plane's bit width.** @@ -437,7 +437,7 @@ no layout consequence. (`arch/x86/repack.cpp:1464`, selected on AVX2 at `repack.cpp:4599-4604`) is what a real llama.cpp model run uses on zen2, and it is an 8-column row-interleaved kernel structurally similar to ours. Any claim of "1.22x ahead on q4_K" should be re-measured against that kernel before it leaves the - building. q3/q5/q6 have no such caveat — llama.cpp has no x86 repack for them. + building. q3/q5/q6 have no such caveat - llama.cpp has no x86 repack for them. 2. **The prefill tiles are 1.6-2.6x ahead on the same formats**, which is consistent with everything above: at tokCount = 4 the per-site unpack (including the redundant high-plane loads) is amortized over four activation dwords instead of one, so the fraction of the loop spent on plane touching From da989abbc11cd8937b8f064625cf34c227c5072a Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:05:08 -0700 Subject: [PATCH 012/123] followup_general 63: the rollback-and-ban sampler for reasoning traces; HOW_TO_GET_SIDECAR.md sections 1-5 as run on the zen4 box Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_GET_SIDECAR.md | 150 +++++++++++++++++++++++++ modules/dasLLAMA/followup_general.md | 12 ++ 2 files changed, 162 insertions(+) create mode 100644 modules/dasLLAMA/HOW_TO_GET_SIDECAR.md diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md new file mode 100644 index 0000000000..54ca6c06d7 --- /dev/null +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -0,0 +1,150 @@ +# How to get a sidecar: a fresh box to a committed CPU-class profile + +You have ssh into a machine nobody has tuned on. This is the walk from there to a checked-in +`performance/defaults/.tune-defaults.json` that every box of that CPU class adopts at start, +with the kernel ladder proving the kernels along the way. Every command below was run as written on +the boxes named in the last section; a step that differs per box says so. + +What a sidecar and a profile are: `skills/tune.md` (the `[tune]` framework, the per-app +`.tune.json`), `../dasLLVM/ARCHITECTURE.md` sec.3 (the knob inventory), `performance/REVIEW.md` +(what a committed profile must satisfy). Short form: the tuner races every kernel family's +`[tune_perm]` seats on the box and writes the winners into the app's sidecar; `export_tune_profile.das` +strips the box-specific rows and saves the kernel winners as the class profile; a box whose +`tune_cpu_class()` resolves to that class adopts the profile and races nothing unless its ISA unlocks +a seat the profile never saw. + +## 1. The box + +CPU classes and the AWS instance that carries each (`us-west-2`; the CLI on the zen2 box is configured): + +| class (`tune_cpu_class()`) | what gates it | instance | +|---|---|---| +| `x86-avx2` | avx2, no VNNI | any zen2/zen3 box (the dev zen2) | +| `x86-vnni512` | avx512vnni + avx512bw | `c7a.4xlarge` (EPYC zen4) | +| `x86-amx` | amx-int8 + amx-tile (+ avx512vnni) | `c7i.4xlarge` (Sapphire Rapids), `c8i` (Granite Rapids) | +| `arm-neon` / `arm-i8mm` | dotprod / i8mm | M1 / M2+, `c8g.2xlarge` (Graviton4) | + +Launch (Ubuntu 24.04, 16 vCPU, 60 GB gp3; the key pair `dasbox` and the group `dasbench-ssh` exist in +the account - the group admits the dev box's IP on port 22, `aws ec2 describe-security-groups +--group-ids sg-0dcc81f768b888767` shows the rule to update when that IP moves): + +``` +aws ec2 run-instances --image-id ami-0ba3fae0cca9442ee --instance-type c7a.4xlarge --key-name dasbox \ + --security-group-ids sg-0dcc81f768b888767 \ + --block-device-mappings '[{"DeviceName":"/dev/sda1","Ebs":{"VolumeSize":60,"VolumeType":"gp3"}}]' \ + --user-data file://aws_bootstrap.sh --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=dasbench-zen4}]' +aws ec2 describe-instances --instance-ids --query 'Reservations[0].Instances[0].[State.Name,PublicIpAddress]' --output text +ssh -i ~/.ssh/dasbox_ed25519 ubuntu@ +``` + +The AMI id is the current Canonical Ubuntu 24.04 amd64 image (`aws ec2 describe-images --owners +099720109477 --filters "Name=name,Values=ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"`, +newest by `CreationDate`). On-demand `c7a.4xlarge` is about $0.82 an hour; terminate when done (last section). + +## 2. Dependencies and the build + +The user-data script (`aws_bootstrap.sh`, reproduced in the last section) does 2-4 unattended; the +same lines by hand: + +``` +sudo apt-get update -y +sudo apt-get install -y --no-install-recommends build-essential clang cmake ninja-build git python3 pkg-config \ + libssl-dev curl ca-certificates libatomic-ops-dev libglu1-mesa-dev freeglut3-dev mesa-common-dev \ + libglfw3-dev libfreetype6-dev libudev-dev libopenal-dev libvorbis-dev libflac-dev libx11-dev \ + libxrandr-dev libxcursor-dev libxinerama-dev libxi-dev +git clone --depth 1 --recursive -b https://github.com/GaijinEntertainment/daScript.git daScript +cd daScript +CC=clang CXX=clang++ cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DDAS_GLFW_DISABLED=ON \ + -DDAS_HV_DISABLED=OFF -DDAS_SQLITE_DISABLED=OFF -DDAS_LLVM_DISABLED=OFF +cmake --build build --target daslang -j 16 +ls -la bin/daslang lib/LLVM.dll +``` + +`-DDAS_LLVM_DISABLED=OFF` is the JIT (default off). The dasLLVM module is das-only; the configure step +downloads the prebuilt `lib/LLVM.dll` for the platform (that name on every OS) - if `ls` does not show it, +nothing below runs. Only `daslang` needs building: dasLLAMA is das, its native needs are builtins. + +## 3. The reference exe + +The kernel ladder's other side. Pin the same tip the dev box's tables use (`6c84c7d5d` for the +2026-09 tables); `test-backend-ops` needs the thread define (a three-line edit of `N_THREADS` in +`tests/test-backend-ops.cpp` - `harness/backend_ops_shapes.patch` is the older, wider form and no +longer applies at this tip): + +``` +git clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp && git checkout 6c84c7d5d +python3 - <<'EOP' +import io +p="tests/test-backend-ops.cpp"; s=io.open(p,encoding="utf-8").read() +old="# define N_THREADS std::thread::hardware_concurrency()" +assert s.count(old)==1 +s=s.replace(old,'# define N_THREADS (getenv("GGML_BENCH_THREADS") && atoi(getenv("GGML_BENCH_THREADS")) > 0 ? (size_t) atoi(getenv("GGML_BENCH_THREADS")) : std::thread::hardware_concurrency())') +io.open(p,"w",encoding="utf-8").write(s) +EOP +cmake -B build-clean-cpu -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON -DLLAMA_CURL=OFF -DLLAMA_BUILD_TESTS=ON +cmake --build build-clean-cpu --target test-backend-ops llama-bench -j 16 +``` + +`build-clean-cpu` is the flavor the tables name: no GPU backend, no BLAS - a GPU build's +`llama-bench -ngl 0` is not CPU-only. + +## 4. Correctness before any number + +``` +cd ~/daScript +DAS_TUNE_MODE=test DAS_JOBQUE_THREADS=8 bin/daslang -jit modules/dasLLAMA/harness/gen_tune_probe.das +``` + +Every `[tune_perm]` seat of every family is stamped and checked bit-exact against its reference body; +the box's ISA is what makes seats that never ran before run here (the 512-bit VNNI seats on zen4, the +AMX leg on Intel). `ok` on every line, or the box has found a defect and the walk stops until it is fixed. + +## 5. The kernel ladder + +``` +LCPP_TBO=~/llama.cpp/build-clean-cpu/bin/test-backend-ops NTOK=512 ROUNDS=5 \ + bash modules/dasLLAMA/harness/kernel_ladder.sh all > ladder.tsv 2> ladder.err +``` + +One thread, no model: every format's stamped kernel against the reference exe's `test-backend-ops perf` +at the same shape, decode and prefill rows, ratio = reference / ours. This is the box's table; it goes +into `plans/kernel_parity_pass.md`'s fact base (and the records store once the kernel board exists). + +## 6. The mint + +(The steps below are being run on the zen4 box as this document is written; they are filled in as +they prove out.) + +- The vehicle: a public 1B GGUF pulled straight to the box, no copying from the dev box - + `curl -L -o Llama-3.2-1B-Instruct-Q4_K_M.gguf https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf` + (808 MB). +- The mint: `bin/daslang -jit modules/dasLLAMA/benchmarks/lcpp_bench.das -- -m --tune` - the + full race of every family on this box (about 316 s on a c7a.4xlarge), writing + `modules/dasLLAMA/benchmarks/lcpp_bench.tune.json`. Read the noise verdict it prints. +- The export: `bin/daslang -jit modules/dasLLAMA/harness/export_tune_profile.das -- --sidecar + modules/dasLLAMA/benchmarks/lcpp_bench.tune.json` writes + `modules/dasLLAMA/performance/defaults/.tune-defaults.json` for this box's + `tune_cpu_class()`; `--class ` overrides the name. +- A class that does not exist yet (Intel's `x86-amx`): the chain lives in + `modules/dasLLVM/daslib/llvm_tune.das` (`tune_cpu_class`, `tune_class_chain`) and every feature a + `requires=` names sits in `TUNE_KNOWN_FEATURES` there; the new class goes above the class it + supersedes so a host resolves to the highest class it satisfies that has a shipped profile. + +## 7. What the commit must satisfy + +`modules/dasLLAMA/performance/REVIEW.md` and its `REVIEW.das` gate: a `defaults/` profile carries no +`engine_sha`, its `provenance.dasllama_version` equals `DASLLAMA_RELEASE`, its `noise` reads `ok`, +and it was minted on a quiet, session-free box - never hand-edited. Run the gate before pushing: +`bin/daslang utils/internal/preflight/main.das -- --only review-md`. + +## 8. Home, and the bill + +``` +scp -i ~/.ssh/dasbox_ed25519 ubuntu@:daScript/modules/dasLLAMA/performance/defaults/.tune-defaults.json . +scp -i ~/.ssh/dasbox_ed25519 ubuntu@:ladder.tsv ladder_.tsv +aws ec2 terminate-instances --instance-ids +``` + +## Boxes this walk ran on + +- 2026-09-01 `c7a.4xlarge` (EPYC 9R14 zen4), `i-043725feb25086523`: sections 1-5 as written. diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 037006eb51..6c653bb70d 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -793,3 +793,15 @@ or fusing the sign flip into the staged slab per SITE via a second indexed table. Done = a form that clears 180 GB/s in the dispatch-loop probe (QUIRK 22's harness), or a note proving the ceiling is shared by llama.cpp's own kernel when isolated the same way. + +63. **Reasoning-trace length under quantization: a rollback-and-ban sampler feature (Boris, + 2026-09-01).** Quantization noise raises the entropy of a reasoning model's intermediate steps, + and a model trained to self-verify treats its own noisier steps as suspect - the "wait, let me + recheck" branch fires more often and each firing adds a hundred tokens. The remedy in the field: + detect the reconsideration n-grams (", but wait", "Wait,", "Hmm,", "Actually," - a table of a few + dozen phrases) after they are emitted, rewind the KV cache to the position before the phrase and + resample with the phrase's first tokens banned. The trigger is multi-token, so a per-token logit + bias cannot express it; the rewind is what makes it exact. Ours: the decode loop owns the position + counter, so the rewind is "n_past back k, drop k cache rows" plus a phrase table and a ban list in + the sampler; expose it as a server/CLI knob. Done = the knob, a test that a forced ", but wait" + stream rewinds and continues, and a before/after token count on a 27B reasoning prompt. From c26e70aec6967dec6e88fc5ceb8594cda9d55c56 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:06:37 -0700 Subject: [PATCH 013/123] kernel parity: the ARM grid memo, its fix on the queue, and the globals emitters' comments say what they emit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 14 +- plans/kernel_parity_pass.md | 16 +- plans/kernel_parity_research_arm.md | 492 ++++++++++++++++++ 3 files changed, 509 insertions(+), 13 deletions(-) create mode 100644 plans/kernel_parity_research_arm.md diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index b877e446ae..35a96101e8 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1481,9 +1481,7 @@ def private emit_kq_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { LLVMPositionBuilderAtEnd(b, cont) } -// The iq3xxs gemv gather globals: the HALVED grid as a [256 x i32] private constant, ksigns -// as a [128 x i32] (7-bit index -> sign byte, bit 7 = parity), plus the shared smask table -// (the same nibble -> byte-mask expansion; magnitudes 2..31 are nonzero, so no carry). +// The iq3xxs gemv gather global: the HALVED grid as a [256 x i32] private constant. def private iq3xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq3xxs.grid") if (grid == null) { @@ -1503,9 +1501,7 @@ def private iq3xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { } -// The iq2s gemv gather globals: the u64 grid as a [2048 x i32] private constant (1024 entries -// as low/high word pairs), plus the shared smask table (magnitudes {8, 25, 43} are nonzero - -// no carry crosses a byte lane on negation). +// The iq2s gemv gather global: the u64 grid as a [2048 x i32] private constant (low/high word pairs). def private iq2s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2s.grid") if (grid == null) { @@ -1525,8 +1521,7 @@ def private iq2s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { } -// The iq2xs gemv gather globals: the u64 grid as a [1024 x i32] private constant (512 entries -// as low/high word pairs), the shared ksigns table, and the shared smask. +// The iq2xs gemv gather global: the u64 grid as a [1024 x i32] private constant (low/high word pairs). def private iq2xs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2xs.grid") if (grid == null) { @@ -1545,8 +1540,7 @@ def private iq2xs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { te.iq3s_grid_g = grid } -// The iq2xxs gemv gather globals: the u64 grid as a [512 x i32] private constant (256 entries -// as low/high word pairs), the shared ksigns table, and the shared smask. +// The iq2xxs gemv gather global: the u64 grid as a [512 x i32] private constant (low/high word pairs). def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var grid = LLVMGetNamedGlobal(mod, "dasllama.iq2xxs.grid") if (grid == null) { diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 8056ea5928..8b9bad51c3 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -158,9 +158,19 @@ CPU decode, the grid formats (done): 16-thread stamp on the vehicles (needs `--for-debug-purposes` on lcpp_bench, or the released exe). 8. retro audit of IQ4_XS/Q3_K per followup 60. -CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x - the grid -decode under the sdot lattice (width 128, two vectors per dword group). Untouched so far; a memo on -llama.cpp's NEON grid kernels vs our width-128 arms is the first step. +CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x. The memo +(`kernel_parity_research_arm.md`): the sdot count is at parity (2 per 32 weights per row, both sides); +the gap is the surrounding work - the parity chain that rebuilds the 8th sign bit for the three ksigns +formats (+3.25 cycles per row per 32 weights: exactly iq2xs/iq2xxs/iq3xxs), the u32 grids' doubled +loads, and the alloca panel, which in decode (tokCount 1) buys back nothing; a port model reproduces +all five M1 numbers within 10%. THE FIX: a 2-rows x 8-weights-per-vector decode form under the sdot +lattice - non-indexed sdot with the activation as `ld1r {v.2d}`, one `addp` per row pair recovers the +`<4 x i32>` accumulator the fold already takes; zero x86 exposure (inside the gather branch, gated on +DOT_SDOT), and it makes llama.cpp's +-1 sign table (`keven_signs`) reachable, retiring the parity chain. +Model: iq2xs ~2570 us = 1.27x. vqtbl gathers are impossible (1-2 KB grids vs tbl's 64 bytes); our +masked negate already beats llama.cpp's own mask expansion. Fallback if deferred: plane-carried sign +bytes (size-neutral for iq2xxs/iq3xxs, +6% for iq2xs; touches the plane, both ISAs) -> 0.72-0.98x only. +Also from the memo: the column-read-vs-byte-read decision was made on x86 and should be a perm seat. Vulkan pp (gap 1): 1. Budget split (measurement, no kernel edit) - is the GEMM the 30%? 2. `shAscales`-style scale diff --git a/plans/kernel_parity_research_arm.md b/plans/kernel_parity_research_arm.md new file mode 100644 index 0000000000..3bead4eaf8 --- /dev/null +++ b/plans/kernel_parity_research_arm.md @@ -0,0 +1,492 @@ +# ARM NEON grid-format decode: what llama.cpp does per 32 weights that we do not + +Scope: the five grid ("codebook") formats on Apple M1 Max, one thread, decode/GEMV +(m=4096, k=14336, one token). Read-only study of +`D:\Work\llama.cpp\ggml\src\ggml-cpu\arch\arm\quants.c` and +`D:\Work\daScript-iquant\modules\dasLLAMA\dasllama\dasllama_gemm_gen.das`. +No file in either tree was modified. + +Measured (us per call, ours vs llama.cpp `test-backend-ops perf -b CPU`): + +| format | ours | llama.cpp | ratio | +|---|---|---|---| +| iq2_xs | 6375 | 3274 | 0.51x | +| iq2_xxs | 6063 | 3437 | 0.57x | +| iq3_xxs | 6735 | 4867 | 0.72x | +| iq3_s | 6089 | 5682 | 0.93x | +| iq2_s | 4932 | 4917 | 1.00x | + +--- + +## 0. Ground facts established first + +**The M1 takes the plain `__ARM_NEON` + dotprod path; there is no i8mm arm for any of the +five.** `__ARM_FEATURE_MATMUL_INT8` appears in `arch/arm/quants.c` only at lines 302, 315, +595, 608, 1155, 1168, 2336, 2360, 2569, 2966, 2984, 3175 — all inside the q4_0/q8_0/q4_K/q5_K +family. The five grid dots start at 3631 (iq2_xxs), 3693 (iq2_xs), 3767 (iq2_s), 3864 +(iq3_xxs), 3926 (iq3_s) and each has exactly two arms: `#if defined(__ARM_NEON)` and the +scalar generic fallback. `ggml_vdotq_s32` is the native `vdotq_s32` when +`__ARM_FEATURE_DOTPROD` is set (`ggml/src/ggml-cpu/ggml-cpu-impl.h:319`), else a +`vmull_s8`+`vpaddlq_s16` emulation (`ggml-cpu-impl.h:310-316`). Our stamped M1 profile +records `"features": "neon;dotprod;fullfp16;lse"` +(`modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json`, provenance block) — +dotprod yes, i8mm no. So both sides run the same primitive: `sdot`. + +**Grid table element widths** (`ggml/src/ggml-common.h`): + +| table | line | element | entries | bytes | weights per entry | +|---|---|---|---|---|---| +| `iq2xxs_grid` | 560 | `uint64_t` | 256 | 2048 | 8 | +| `iq2xs_grid` | 627 | `uint64_t` | 512 | 4096 | 8 | +| `iq2s_grid` | 758 | `uint64_t` | 1024 | 8192 | 8 | +| `iq3xxs_grid` | 1017 | `uint32_t` | 256 | 1024 | 4 | +| `iq3s_grid` | 1052 | `uint32_t` | 512 | 2048 | 4 | +| `ksigns_iq2xs` | 513 | `uint8_t` | 128 | 128 | (7-bit code -> sign byte) | +| `ksigns64` | 524 | `uint64_t` | 128 | 1024 | (7-bit code -> 8 x 0x00/0xFF) | +| `keven_signs_q2xs` | `arch/arm/quants.c:3595` | `int8_t[1024]` | 128 x 8 | 1024 | (7-bit code -> 8 x +/-1) | + +`keven_signs_q2xs` is the ARM-only table: 128 entries of eight `+1`/`-1` bytes, read as +`uint64_t` through `signs64` (`quants.c:3646, 3708, 3879`). It is a **pre-expanded +/-1 +vector**, so applying signs is one `vmulq_s8` with zero bit manipulation. This single table +is the largest structural difference in the whole comparison. + +**Our shape on NEON.** `setup_tile_emit` pins `te.width = 128` on the sdot leg +(`dasllama_gemm_gen.das:1900`), `te.rv = width/32 = 4`, `te.rq = interleave/rv` +(1901-1902). The stamped `mr` is 8 for all five formats on ARM +(`arm-neon.tune-defaults.json`: `iq2sq8_tile_gen`..`iq3xxsq8_tile_gen` all `"mr8"`), so +rv = 4 rows per 16-byte vector, rq = 2 vectors per 8-row group. `sdot_lane` +(`dasllama_gemm_gen.das:309-316`) builds the **indexed** form: the activation is bitcast to +`v4i32`, one dword lane splatted, and the backend folds that into `sdot v,v,v[lane]`. So one +sdot = 4 rows x 4 weights = 16 byte-products. + +**Both sides issue the same number of sdot per weight.** 32 weights x 1 row = 32 +byte-products = 2 sdot for llama.cpp (`quants.c:3747-3750` etc.). Ours: per block per qd, +`for j in 0..3` x lo/hi = 8 sdot covering 4 rows x 32 weights = 128 byte-products, i.e. +2 sdot per row per 32 weights. **The dot is at exact parity. Everything below is the +surrounding work.** + +**Our GEMV re-uses nothing.** `kq_gemv_gen_impl` drives `emit_slice(te, sa, ghead, 0, 1, "")` +(`dasllama_gemm_gen.das:2880`) — tokCount = 1. Every decoded weight is consumed by exactly +one sdot. The alloca panel (`LLVMBuildAlloca(..., 256 * te.interleave, "iq3s.panel")`, line +2845 — 2048 bytes at mr=8) therefore buys **no reuse whatsoever in decode**; it exists only +to separate the rolled gather loop from the unrolled dot body. The prefill/tile path is the +one that benefits, and there the panel is produced once per group by +`unpack_kq_panel_grp` (`dasllama_repack.das:953`) and amortized over 4 tokens — which is +exactly why every prefill tile is 1.9x-15x ahead while decode is behind. + +--- + +## 1. Per-format op budget, side by side + +Counting is per **one row x 32 weights** (llama.cpp's natural unit; ours divided down from +the 4-rows-per-vector shape). "mem" = load/store uops, "NEON" = vector ALU uops, "int" = +scalar integer ALU uops. + +### 1.1 llama.cpp + +**iq2_xxs** (`quants.c:3660-3680`, body covers 64 weights) +- `ggml_vld1q_s8_x4(q8)` (3662): 4 x 16B activation loads / 64w = **2 mem** +- `memcpy(aux32, q2, 16)` (3663): 2 x `ldr x` / 64w = **1 mem** +- grid: 8 x `vld1_s8(iq2xxs_grid + aux8[i])` (3664-3667), paired by `vcombine_s8` into + `ldr d` + `ld1 {v.d}[1]` = **4 mem** + 4 byte-extracts = **4 int** +- signs: 8 x `vld1_s8(signs64 + ((aux32[k] >> s) & 127))` (3668-3671) = **4 mem** + + 4 `ubfx` = **4 int** +- `vmulq_s8` x4 (3672-3675) = **2 NEON** +- `ggml_vdotq_s32` x4 (3676-3677) = **2 NEON** +- epilogue `vaddvq_s32` + float scale (3678-3679) ~ **3 mixed** + +**Total: 11 mem, 4 NEON, 8 int, ~3 epilogue ~ 26 uops.** + +**iq2_xs** (`quants.c:3733-3753`, body covers 64 weights) +- q8 (3734) = **2 mem** +- grid: 8 x `vld1_s8(iq2xs_grid + (q2[i] & 511))` (3735-3738) = 4 index `ldrh` + 4 `and` + + 4 grid `ldr d` (the `vcombine` pair is `ldr d` + `ld1 {v.d}[1]`, so 8 half-loads per + 64w = 4 per 32w) -> **8 mem + 4 int** +- signs: 8 x `vld1_s8(signs64 + (q2[i] >> 9))` (3739-3742) = **4 mem + 4 int** (the + `ldrh` is shared with the grid lookup) +- `vmulq_s8` x4 (3743-3746) = **2 NEON** +- `vdotq` x4 (3747-3750) = **2 NEON** +- `vpaddq_s32` x3 + `vmlaq_s32` (3751-3752) = **2 NEON** +- per-superblock scale prep (3719-3729) amortized over 8 blocks ~ **2** + +**Total: 14 mem, 6 NEON, 8 int ~ 28-30 uops.** + +**iq2_s** (`quants.c:3808-3848`) +- q8 (3809) = **2 mem** +- grid: 8 x `vld1_s8(iq2s_grid + (qs[i] | ((qh[..] << s) & 0x300)))` (3810-3817) = 4 `ldrb` + + 4 grid `ldr d` = **8 mem**, plus `<<`/`&`/`|` x3 per index = **12 int** +- signs are **plane bits**, expanded with the mask1/mask2 shuffle (3820-3824, 3829-3833): + per 32 weights = 1 `vdupq_n_u32` + 2 `vqtbl1q_u8` + 2 `vandq` + 2 `vceqq` = **7 NEON** +- `vorrq_u8(vs, m1)` x2 + `vmulq_s8` x2 (3826-3827) = **4 NEON** +- `vdotq` x2 (3840-3843) = **2 NEON** +- 2 x (`vaddvq_s32` + nibble scale extract + `madd`) (3845-3848) ~ **6 mixed** + +**Total: 10 mem, 13 NEON, 12 int ~ 37 uops.** llama.cpp's most expensive of the five, and +indeed its slowest but one. + +**iq3_xxs** (`quants.c:3894-3912`) +- q8 (3894) = **2 mem**, `memcpy(aux32, gas, 8)` (3895) = **0.5 mem** +- grid: `ggml_vld1q_u32(iq3xxs_grid[q3[0]], .., [q3[3]])` (3896-3899). That macro is a + plain brace initializer on non-MSVC (`ggml-cpu-impl.h:84`), so clang emits 4 index `ldrb` + + 4 `ldr w` + 3 lane inserts per q register, 4 registers per 64 weights -> + **16 mem + 12 int per 64w = 8 mem + 6 int per 32w** +- signs: 8 x `vld1_s8(signs64 + ...)` (3900-3903) = **4 mem + 4 int** +- `vmulq_s8` x4 (3905-3908) = **2 NEON**, `vdotq` x4 (3909-3910) = **2 NEON** +- epilogue (3911-3912) ~ **4 mixed** + +**Total: 14.5 mem, 4 NEON, 10 int ~ 40 uops.** + +**iq3_s** (`quants.c:3983-4022`) +- q8 (3983) = **2 mem** +- index vector: `vld1q_u8(qs)` (3985) then `vmovl_u8`/`vshlq_u16`/`vandq`/`vorrq` + (3986, 3991) = **0.5 mem + ~6 NEON per 32w** +- **the `vec_index_t` union (3941-3944)**: `idx.vec_index` is written as a vector and then + read back as `idx.index[0..7]` scalars (3987-3990, 3992-3995). That is a store-forwarding + round trip through the stack — 1 vector store + 8 `ldrh` per half, **~9 mem per 32w** +- grid: 8 x `ldr w` + 6 lane inserts per 32w = **8 mem + 6 int** +- signs: the mask1/mask2 expansion (3998-4002, 4007-4011) = **7 NEON**, plus `vmulq_s8` x2 + (4004-4005, 4015-4016) = **2 NEON** +- `vdotq` x2 (4018-4019) = **2 NEON**, epilogue (4021-4022) ~ **3** + +**Total: ~19.5 mem, ~17 NEON, ~6 int ~ 49 uops.** llama.cpp's slowest of the five, matching +its 5682 us. + +### 1.2 Ours + +Two loops per superblock, both walked per row group of mr = 8: + +**(A) `emit_kq_gather` (`dasllama_gemm_gen.das:1451-1486`)** — a rolled loop over the mr +rows; each iteration emits **64 straight-line dword decodes** (`for blk in 0..7`, +`for l in 0..3`, `for h in 0..1`, line 1470-1477), each decoding **4 weights of one row** +and storing them with `LLVMBuildStore` into the panel at `panel_group_off` (1443-1449, +1478-1479). + +**(B) the dot body inside `emit_block_iq4xs` (1573-1712)** — per block: 2 activation +`v16i8` loads shared across all 8 rows (1620-1621); per (blk, qd) one `kq_sign_bytes` +(1632); per (blk, j, qd) two panel `v16i8` loads (1642-1643), two `apply_sign_col` +(1645-1646), two `dot_lane` (1659-1662). + +Per **one row x 32 weights**, gather side: + +| format | decode site | mem uops | int uops | +|---|---|---|---| +| iq2_xs | `decode_iq2xs_dword` 1358-1367 | 2 column `ldr w` (`wg_col_byte` 1308-1315) + 4 grid `ldr x` (`grid_pair_half` 1324-1330) + **8 panel `str w`** = **14** | 12 byte extracts + 8 `orr/lsl` for the u16 + 4 `and #511` + 4 `lsr #32` = **28** | +| iq2_xxs | `decode_iq2xxs_dword` 1369-1373 | 4 `ldrb` (`wg_byte` 1293-1296) + 4 grid `ldr x` + **8 `str w`** = **16** | ~8 | +| iq2_s | `decode_iq2s_dword` 1347-1356 | 1 qh `ldrb` + 4 qs `ldrb` + 4 grid `ldr x` + **8 `str w`** = **17** | ~16 | +| iq3_xxs | `decode_iq3xxs_dword` 1342-1344 | 2 column `ldr w` + 8 grid `ldr w` (`grid_word` 1317-1321) + **8 `str w`** = **18** | ~12 | +| iq3_s | `decode_iq3s_dword` 1332-1340 | 1 qh `ldrb` + 8 qs `ldrb` + 8 grid `ldr w` + **8 `str w`** = **25** | ~24 | + +Per **one row x 32 weights**, dot side (shared shape, divided by 4 rows per vector): + +- panel loads: 8 x `ldr q` / 4 rows = **2 mem** +- activation: 2 x `ldr q` / 8 rows = **0.25 mem** +- `apply_sign_col` (1416-1429): the `idx` shuffle depends only on `l`, so 4 distinct `tbl` + per (blk, qd) plus 8 x (`and` + `cmpeq` + `eor` + `sub`) = 36 NEON / 4 rows = **9 NEON** +- `sdot`: 8 / 4 rows = **2 NEON** +- epilogue (sub-scale load, `mul`, `add`, 1683-1706) ~ **1.5 NEON** +- `kq_sign_bytes` (1378-1414): + - **iq2_s (kq 23) and iq3_s (kq 33)**: the plane carries the sign bytes, so it is + **one `ldr q`** (1385-1388) = **0.25 mem, 0 NEON** + - **iq2_xs (24), iq2_xxs (25), iq3_xxs (34)**: 1-2 `ldr q` plus, for each of 4 sign + groups, code extraction + a **3-step parity chain** (`lshr`/`xor` x3, lines 1404-1406) + + `and`/`shl`/`orr` + the byte pack = ~13 NEON x 4 = **53 NEON / 4 rows = 13.25 NEON** + +**This is the single biggest asymmetry.** llama.cpp never computes a parity: it indexes +`keven_signs_q2xs` (`quants.c:3595`), a table whose entries already carry the eighth, +parity-implied sign. We recompute the eighth sign in registers, for every sign group, of +every block, of every superblock, of every row. + +### 1.3 A port model that fits the measurements + +M1 Firestorm: 8-wide issue, ~3 load units, ~2 store units (~3 memory uops/cycle sustained), +4 NEON pipes, ~6 integer ALU pipes, 3.22 GHz. Modelling each loop as +`max(mem/3, NEON/4, int/6)` cycles per row per 32 weights: + +| format | gather cyc | dot cyc | total cyc | predicted us | measured us | error | +|---|---|---|---|---|---|---| +| iq2_s | 5.7 | 3.2 | 8.9 | 5070 | 4932 | +3% | +| iq2_xs | 4.7 | 6.45 | 11.15| 6354 | 6375 | 0% | +| iq2_xxs | 5.3 | 6.45 | 11.75| 6696 | 6063 | +10% | +| iq3_s | 8.3 | 3.2 | 11.5 | 6553 | 6089 | +8% | +| iq3_xxs | 6.0 | 6.45 | 12.45| 7094 | 6735 | +5% | + +(predicted us = cycles x 1.835e6 row-32w units x 1e6 / 3.22e9; 4096 x 14336 / 32 = 1.835e6.) + +**The model tracks all five within 10%, which means our kernel is genuinely port-saturated.** +The same model applied to llama.cpp under-predicts by 20-50% (iq2_xs: 4.67 cyc -> 2661 us vs +3274 measured; iq3_s: 6.5 cyc -> 3705 vs 5682), because llama.cpp's serial +index-load -> grid-load chains and per-32 float accumulation leave it latency-bound with +issue slack. Practical consequence: **op count is our currency and only our currency; we +cannot buy time back with better scheduling, and llama.cpp's measured times are an upper +bound on what its op mix actually costs.** + +The model also reproduces the *ranking* exactly, and it decomposes into two independent +additive penalties: + +- **+3.25 cyc/row/32w** if the format's signs are ksigns codes rather than plane bytes + (iq2_xs, iq2_xxs, iq3_xxs). +- **+2.6 cyc/row/32w** if the grid is `u32` (4 weights/entry) rather than `u64` + (8 weights/entry), because the gather issues twice the grid loads (iq3_s, iq3_xxs). + +iq2_s pays neither and sits at 1.00x. iq3_s pays only the second and sits at 0.93x. The +three at 0.51x-0.72x are exactly the three that pay the parity chain. **The 0.51x/0.57x +outliers are not "ARM is hard"; they are one identifiable code path.** + +--- + +## 2. The deltas, ranked by decode payoff on M1 + +Each entry is tagged **emitter change** / **repack change** / **seat**, with its x86 +exposure. Zen2 is at 0.99x-1.32x on these five, so nothing here may touch the AVX2 lattice +without a re-measure. + +### D1 — the gather decodes one dword per scalar step and round-trips 32 B/row/block through an alloca + +**Payoff: the largest. Tag: emitter change (GEMV-only, sdot-leg-only). x86: none.** + +`emit_kq_gather` decodes 4 weights at a time (`decode_kq_dword`, 1431-1438) and stores them +as a 4-byte `str` (1478-1479); the dot body then reloads them as `v16i8` +(1642-1643). Per row per 32 weights that is **8 stores + 2 loads = 10 of the ~14-25 memory +uops** in the gather. Since tokCount = 1 in decode (line 2880), those 2 KB are written and +read back once, for nothing. + +Worse, the 4-byte granularity is *forced* by the panel's element order. For the u64-grid +formats one grid entry is 8 consecutive weights `l*8 .. l*8+7`, but +`panel_group_off` (1443-1449) sends `e = l*8` and `e = l*8+4` to different dword groups +`mr*4 = 32` bytes apart (the same split is visible in the tile's unpack: +`dasllama_repack.das:896-901`, `d1`/`d2` computed separately, then two 4-byte +`wp1[0]`/`wp2[0]` stores). So a single 8-byte grid load costs two 4-byte stores plus a +`lsr #32` to split it (`grid_pair_half`, 1324-1330). + +llama.cpp never splits: `vcombine_s8(vld1_s8(grid+i), vld1_s8(grid+j))` (3735) puts two +whole u64 entries — 16 consecutive weights of one row — into one q register with two loads +and no store at all. + +### D2 — the three ksigns formats synthesize the eighth sign bit with a 3-step parity chain, per group, per block, per row + +**Payoff: 3.25 cyc/row/32w, ~29% of iq2_xs's time. Tag: emitter change (if D1 lands) or +repack change (standalone). x86: shared code, must re-measure zen2.** + +`kq_sign_bytes` lines 1400-1412: `code` -> `par = code ^ (code>>4)`, `par ^= par>>2`, +`par ^= par>>1`, `sb = code | ((par&1)<<7)`, then a 4-way byte pack. 13 vector ops per sign +group, 4 groups per block, on `<4 x i32>` — 53 NEON uops per (4 rows, 32 weights). + +llama.cpp: `vld1_s8(signs64 + code)` — one load of a pre-expanded +/-1 vector +(`quants.c:3739-3742`, table at 3595), then one `vmulq_s8`. The parity is baked into the +table. + +Note the comment drift here: lines 1484-1485, 1507, 1529 and 1549 all describe "the shared +ksigns table" and "the shared smask table" as emitted globals, but the `*_emit_globals` +functions only create the grid global — no ksigns or smask constant is emitted, and +`kq_sign_bytes` reads none. Those four comment fragments are stale. + +### D3 — the u64 grid's 8 weights are decoded as two separate 4-weight steps + +**Payoff: it is what makes D1's 8 stores 8 instead of 4, and it doubles the index work for +iq2_xs. Tag: emitter change. x86: same code path, gated by the same `gather` flag.** + +`grid_pair_half` (1324-1330) issues one 8-byte load and then `trunc`s the low or high half +per call. LLVM's CSE folds the two calls onto one load, but the *consumers* are two +independent 4-byte stores at unrelated panel offsets, so the pair can never become a single +8-byte store. + +### D4 — the packed plane's 4-byte column layout costs a dword load plus shift and mask per index byte + +**Payoff: ~2 cyc/row/32w on iq2_xs specifically. Tag: repack change or seat. x86: the +current shape was chosen ON x86.** + +`wg_col(mr, pos) = (pos/4)*mr*4 + pos%4` (1304) interleaves rows in 4-byte columns, so +`wg_col_byte` (1308-1315) reads a dword and shifts the byte out. For iq2_xs the 16-bit qs +word is then rebuilt from two such bytes (`decode_iq2xs_dword`, 1358-1367: `qlo | (qhi<<8)`), +where llama.cpp does one `ldrh` (3735). That is 12 extraction + 8 reassembly ops against +llama.cpp's 4 loads. + +The doc-comments on 1342 and 1358 record the measurements that chose this shape +("the column read measured 7862 -> 6602 us", "6501 -> 4702 us; one u16 load measured slower +than two bytes"). Those figures are from the x86 development box, and the same comment on +1342 already admits the column read *lost* for iq3s/iq2s/iq2xxs. **The decode shape is +currently one global decision made on one ISA.** It should be a seat. + +### D5 — the sign application is a 5-op masked negate rather than one multiply + +**Payoff: small on its own (~1 cyc/row/32w), large once D1 lands. Tag: emitter change.** + +`apply_sign_col` (1416-1429): `tbl` broadcast of the row's sign byte, `and` with a per-lane +bit constant, `cmpeq`, `eor`, `sub`. Against llama.cpp's `vmulq_s8` with a table-loaded ++/-1 vector. Our form costs 4 ALU ops per weight vector plus a shared `tbl`; llama.cpp's +costs 1 multiply plus the 2 loads already counted. + +But this is a *consequence*, not a cause: our 16-byte vector holds **4 different rows**, so +a +/-1 vector for it would need four separate table entries and four lane inserts. The +masked negate is the correct choice given the current layout. It is only fixable by changing +the layout (D1/section 3a). + +Note the cost is already close to llama.cpp's for iq2_s/iq3_s: our 9 NEON/row/32w against +llama.cpp's 11 for the mask1/mask2 expansion (`quants.c:3820-3827`). This is precisely why +iq2_s measures 1.00x. + +--- + +## 3. The three proposals evaluated + +### (a) A width-128 gather composing u64 grid entries as 64-bit lanes: 2 rows x 8 weights per vector + +**Verdict: this is the fix. Adopt it.** + +Layout: vector `W = [ row A weights e..e+7 | row B weights e..e+7 ]`, one 16-byte register +from two 8-byte grid loads (`ldr d` into lane 0, `ld1 {v.d}[1]` into lane 1 — in LLVM, a +load plus an `insertelement` into a `<2 x i64>`). + +The dot then uses the **non-indexed** `sdot Vd.4S, Vn.16B, Vm.16B`, whose lane i is +`Vn[4i..4i+3] . Vm[4i..4i+3]`. Feed it `X = [ x(e..e+3) | x(e+4..e+7) | x(e..e+3) | +x(e+4..e+7) ]`, i.e. the 8 activation bytes replicated — exactly `ld1r {v.2d}, [xp]`, one +instruction, no lane-splat shuffle at all. Then: + +- lane 0 = row A . x(e..e+3), lane 1 = row A . x(e+4..e+7) +- lane 2 = row B . x(e..e+3), lane 3 = row B . x(e+4..e+7) + +Accumulate all 4 element-groups of a block into **one** `<4 x i32>` per row pair; at block +end, one `addp` of two row-pair accumulators yields `[rowA, rowB, rowC, rowD]` — **the exact +`<4 x i32>` shape `a[]` already has** (line 1659-1662), so the entire scale/fold epilogue +(1683-1711) is untouched. `pairwise_add_i32` (909-917) already emits that `addp`. + +Products per sdot: 2 rows x 8 weights = 16, identical to today's 4 x 4. **No dot regression.** + +Same intrinsic as today (`llvm.aarch64.neon.sdot`, `<4 x i32>`/`<16 x i8>`, declared at +1911-1918) — only the operand construction changes, so no new decline rail. + +Projected budget per **2 rows x 32 weights** for iq2_xs, keeping the panel: + +| | uops | +|---|---| +| index loads (a row pair's 4+4 index bytes are contiguous in the column -> one `ldr x`) | 2 mem | +| grid: 8 u64 loads | 8 mem | +| signs: 8 loads from a +/-1 table, indexed by the 7-bit code | 8 mem | +| `vmulq_s8` x4 | 4 NEON | +| panel: 4 x 16-byte stores | 4 mem | +| dot: 4 panel loads + 4 sdot + 1 `ld1r` | 5 mem, 4 NEON | +| index extraction | ~16 int | + +mem 27 -> 9 cyc, NEON 8 -> 2 cyc, int 16 -> 2.7 cyc. **9 cyc per 2 rows = 4.5 cyc/row/32w** +against today's 11.15. At the fitted 572 us/cycle-unit that is **iq2_xs ~2570 us, i.e. +1.27x AHEAD of llama.cpp's 3274**. Even a 50% miss lands at parity. + +If the gather is *fused* into the dot body (legitimate in decode, where tokCount = 1 and the +panel buys nothing), the 4 stores and 4 loads vanish: ~19 mem -> 6.3 cyc per 2 rows = +**3.2 cyc/row/32w, ~1810 us**. Cost: the superblock body unrolls to 4 row-pairs x 32 +element-groups x ~6 ops ~ 770 instructions, which is actually *smaller* than today's +combined dot body (~900) plus gather body. Worth trying second. + +Notes and caveats: + +- **The +/-1 sign table replaces D2 entirely.** Under this layout the sign vector for a row + pair is two 8-byte loads from a `keven_signs`-equivalent global indexed by the 7-bit code — + no parity chain, no plane change, no repack. D2 becomes unnecessary if (a) lands. +- **For iq2_s / iq3_s** (signs are plane *bits*, not codes) emit a 256-entry x 8-byte +/-1 + table indexed by the raw sign byte (2 KB private constant, same shape as the grid globals + at 1273-1291) so all five formats share one sign path. +- **For the u32-grid formats** (iq3_xxs, iq3_s) a row's 8 weights are two grid entries, so + building the half costs 2 loads (`ldr s` + `ld1 {v.s}[1]`) instead of 1 — 16 grid loads per + (2 rows, 32w) instead of 8. Projected ~5 cyc/row/32w, still ~2.3x better than today and + ahead of llama.cpp, which pays the same 8 u32 loads per row plus 6 lane inserts + (`quants.c:3896-3899`, `ggml-cpu-impl.h:84`). +- **Blast radius is small and ARM-only.** The panel is the GEMV's private alloca (line 2845), + the new shape lives entirely inside the `if (gather)` branch (line 1581), and the whole + thing is gated on `te.dotKind == DOT_SDOT`. **No plane change, no repack change, no tile + change, no x86 exposure.** Prefill (already 1.9x-15x ahead) is untouched. +- **Make it a seat for one tuning round.** Add a perm token (`gather="pair"` vs the current + `"dword"`) parsed in `parse_perm` (101-125) so the M1 sidecar proves it per format before + it becomes the default. The existing seats are one string per format + (`iq2xsq8_tile_gen => "mr8"`), so the token composes with `mr8` without disturbing x86's + `dot_maddubs_width256_mr8`. + +### (b) ARM gather via vector table lookup (`vqtbl`) + +**Verdict: impossible for the grid. Not worth an experiment.** + +NEON `TBL`/`TBX` index into 1 to 4 *consecutive V registers* — a 16, 32, 48 or 64-byte +table, with out-of-range indices producing zero. The smallest grid here is `iq3xxs_grid`, +256 x `uint32_t` = **1024 bytes** (`ggml-common.h:1017`) — 64 V registers, 16x the ISA +maximum. `iq2xxs_grid` is 2048 bytes (`ggml-common.h:560`). There is no NEON gather +instruction of any kind, and the M1 has no SVE (profile features string: +`neon;dotprod;fullfp16;lse`). So the grid must be read with scalar loads on this ISA, and +llama.cpp does exactly that (`vld1_s8`/`ldr w` per entry) for the same reason. + +Where `tbl` *is* the right tool, and where both sides already use it: + +- the 16-entry nibble codebook for iq4_nl/iq4_xs — we already emit `llvm.aarch64.neon.tbl1` + (`dasllama_gemm_gen.das:1929-1936`, `lut_lookup` 632-642) +- broadcasting a sign byte across its four lanes — `apply_sign_col`'s constant + `shufflevector` (1421-1422) lowers to `tbl`, and llama.cpp's `mask1` shuffle + (`quants.c:3822`, `4000`) is the same instruction + +No format in this set has a grid small enough for a `vqtbl4q` sub-table. + +### (c) Sign application: `vmulq_s8` with a table-derived sign vector vs our and+cmpeq+xor+sub + +**Verdict: our masked negate is not the defect. The defect is the sign *source*, and the +multiply form is unreachable without (a).** + +Cost accounting, per one row x 32 weights: + +| scheme | ops | +|---|---| +| llama.cpp, ksigns formats (`quants.c:3739-3746`): 4 `vld1_s8` + 2 `vmulq_s8` | 4 mem + 2 NEON | +| llama.cpp, plane-bit formats (`quants.c:3820-3827`): dup + 2 tbl + 2 and + 2 ceq + 2 orr + 2 mul | 11 NEON | +| ours, `apply_sign_col` (1416-1429): 4 tbl + 8 x (and, cmpeq, eor, sub), / 4 rows | 9 NEON | +| ours, `kq_sign_bytes` ksigns arm (1400-1412): 53 NEON / 4 rows | 13.25 NEON | +| ours, `kq_sign_bytes` plane arm (1385-1388): 1 load / 4 rows | 0.25 mem | + +So our *application* (9) already beats llama.cpp's bit expansion (11) and is within reach of +its multiply form (2 NEON + 4 mem). The 13.25 NEON of *code-to-byte synthesis* is the whole +problem, and it exists only because the format stores a 7-bit code where iq2_s and iq3_s +store a byte. + +Two ways to kill it: + +1. **Under (a)**: index a +/-1 table with the 7-bit code and multiply. Zero synthesis, zero + plane change. This is llama.cpp's move, and it is only available once a vector holds + whole 8-weight runs of a *single* row (a 4-row-interleaved vector would need 4 separate + table entries plus 4 lane inserts, which is why the masked negate was correct until now). +2. **Standalone (repack change, if (a) is deferred)**: store the expanded 8-bit sign byte in + the plane instead of the 7-bit code, turning iq2_xs/iq2_xxs/iq3_xxs into the iq2_s/iq3_s + shape. + - **iq2_xxs**: the aux32 column already holds 4 x 7-bit codes plus a 4-bit `ls` in 4 + bytes, and the scale already lives in the scale plane (`IQ2XXS_SSB` comment, + `dasllama_kqformat.das:118`), so 4 sign bytes fit in the same 4 bytes. + **Size-neutral** (`IQ2XXS_QSB` stays 64, line 117). + - **iq3_xxs**: identical — 8 blocks x 4 aux bytes -> 8 x 4 sign bytes. + **Size-neutral** (`IQ3XXS_QSB` stays 96, line 111). + - **iq2_xs**: the 9-bit index does not fit a byte. Restructure to iq2_s's shape — + 32 index low bytes + 4 high-bit bytes + 32 sign bytes = 68 B against today's 64 + (`IQ2XS_QSB`, line 116). **+6%.** + Projected under this alone: iq2_xs 6375 -> ~4520 (0.72x), iq2_xxs 6063 -> ~4390 (0.78x), + iq3_xxs 6735 -> ~4980 (0.98x). Real, but short of parity for the iq2 pair, which is why + (a) is the recommendation and this is the fallback. + + **x86 caution for option 2**: it changes the on-disk plane, the repack, the tile unpack + (`dasllama_repack.das:874-949`) and both ISAs. The +6% on iq2_xs is affordable on + bandwidth grounds — the decode moves 16 MB per call in 6375 us = 2.5 GB/s against M1 + Max's tens of GB/s single-core, so both implementations are ~15-25x off the memory roof + and firmly compute-bound — but zen2's iq2_xs must be re-measured before it lands. iq2_xxs + and iq3_xxs being size-neutral carry no bandwidth risk on either ISA. + +--- + +## 4. Recommended order + +1. **(a) as a seat**, u64-grid formats first (iq2_xs, iq2_xxs, iq2_s), panel retained, + signs from an emitted +/-1 table. Emitter-only, GEMV-only, sdot-leg-only, no x86 + exposure. Model says 0.51x -> ~1.27x on iq2_xs; measure before believing. +2. **(a) extended to the u32-grid formats** (iq3_xxs, iq3_s) with the 2-loads-per-half + build. Model says ~5 cyc/row/32w, comfortably ahead. +3. **Fuse the gather into the dot body** (drop the panel in decode only) once (a) is + stamped. Model says a further ~30%. Watch generated code size. +4. **Make the decode shape a seat** regardless (D4): the column-read-vs-byte-read choice at + lines 1342 and 1358 was measured on x86 and is currently binding on ARM. +5. **Only if (a) is abandoned**: the plane sign-byte repack for iq2_xxs and iq3_xxs + (size-neutral, safe on both ISAs), and separately iq2_xs (+6%, needs a zen2 re-measure). + +Incidental finding for whoever touches these lines: the "shared ksigns table" and "shared +smask table" described in the comments at `dasllama_gemm_gen.das:1484-1485, 1507, 1529, 1549` +are not emitted by the functions those comments head, and `kq_sign_bytes` reads neither. From ec2357ab01918bbdf756934b8556e5ccb02b0a0f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:07:50 -0700 Subject: [PATCH 014/123] kernel parity: the zen4 ladder - 65/65 TEST, VNNI seats pay on the dot-bound formats, the grid gather is the scalar bound on both ISAs Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- plans/kernel_parity_pass.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 8b9bad51c3..481e5db34d 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -80,6 +80,14 @@ reference / ours): decode q8 1.46, k4 1.60, k5 1.06, k6 1.35, q40 1.96, q51 1.32 every tile 1.9x-15.3x ahead. On ARM the grid formats are the tails - the reference exe's NEON grid kernels run iq2xs/iq2xxs at 3.3-3.4 ms against our 6.0-6.4 ms while the k-quants are 1.06x-1.60x. +The zen4 ladder (AWS c7a.4xlarge, EPYC 9R14, one thread, the x86-vnni512 profile's seats = +dot_vpdpbusd_width512_mr16; gen_tune_probe TEST 65/65 ok on the first run of those seats): decode q40 +3.87, iq4nl 2.82, q51 2.65, mx4 2.51, k4 2.25, iq4xs 2.13, k2 2.04, k3 1.88, q8 1.70, k5 1.28, k6 0.91, +iq3xxs 0.88, iq3s 0.81, iq2xxs 0.81, iq2xs 0.79, iq2s 0.77; every tile 2.1x-12x ahead. Where the dot is +the work, the 512-bit VNNI seat pays (k4 1.22x on zen2 -> 2.25x); the five grid formats lose ground +because the reference's grid kernels gain ~1.5x from AVX-512 while ours are bound by the scalar per-dword +gather (iq2s 4898 us on zen4 vs 5059 on zen2) - the ARM memo's diagnosis, on x86. + zen2 reading: every tile row is ahead (1.17x-7.07x). Decode tails were k3 0.80x, k6 0.90x, k5 0.91x; iq3xxs, k2, iq2s, iq2xxs at 0.99-1.00 (inside the noise band). The 4-bit class (q40, q51, iq4xs, iq4nl, k4, q8, mx4) sits at 1.2x-1.6x. @@ -158,6 +166,12 @@ CPU decode, the grid formats (done): 16-thread stamp on the vehicles (needs `--for-debug-purposes` on lcpp_bench, or the released exe). 8. retro audit of IQ4_XS/Q3_K per followup 60. +CPU decode, grid formats, BOTH ISAs (M1 0.51-0.93x, zen4 0.77-0.88x): the scalar per-dword gather is +the bound; the fix is the vectorized u64-lane compose (2 rows x 8 weights per NEON vector, 4 per 256-bit, +8 per 512-bit) with one pairwise add recovering the accumulator layout - one emitter change under the +gather branch, no plane change, retires the panel and the parity chain. k6 0.90-0.91 on both x86 boxes +stays open. + CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x. The memo (`kernel_parity_research_arm.md`): the sdot count is at parity (2 per 32 weights per row, both sides); the gap is the surrounding work - the parity chain that rebuilds the 8th sign bit for the three ksigns From 78c193cf6a76f78b201415615a638a8177bc5423 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:08:12 -0700 Subject: [PATCH 015/123] the ARM memo through the Markdown ASCII gate Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- plans/kernel_parity_research_arm.md | 62 ++++++++++++++--------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/plans/kernel_parity_research_arm.md b/plans/kernel_parity_research_arm.md index 3bead4eaf8..0729cfa936 100644 --- a/plans/kernel_parity_research_arm.md +++ b/plans/kernel_parity_research_arm.md @@ -22,14 +22,14 @@ Measured (us per call, ours vs llama.cpp `test-backend-ops perf -b CPU`): **The M1 takes the plain `__ARM_NEON` + dotprod path; there is no i8mm arm for any of the five.** `__ARM_FEATURE_MATMUL_INT8` appears in `arch/arm/quants.c` only at lines 302, 315, -595, 608, 1155, 1168, 2336, 2360, 2569, 2966, 2984, 3175 — all inside the q4_0/q8_0/q4_K/q5_K +595, 608, 1155, 1168, 2336, 2360, 2569, 2966, 2984, 3175 - all inside the q4_0/q8_0/q4_K/q5_K family. The five grid dots start at 3631 (iq2_xxs), 3693 (iq2_xs), 3767 (iq2_s), 3864 (iq3_xxs), 3926 (iq3_s) and each has exactly two arms: `#if defined(__ARM_NEON)` and the scalar generic fallback. `ggml_vdotq_s32` is the native `vdotq_s32` when `__ARM_FEATURE_DOTPROD` is set (`ggml/src/ggml-cpu/ggml-cpu-impl.h:319`), else a `vmull_s8`+`vpaddlq_s16` emulation (`ggml-cpu-impl.h:310-316`). Our stamped M1 profile records `"features": "neon;dotprod;fullfp16;lse"` -(`modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json`, provenance block) — +(`modules/dasLLAMA/performance/defaults/arm-neon.tune-defaults.json`, provenance block) - dotprod yes, i8mm no. So both sides run the same primitive: `sdot`. **Grid table element widths** (`ggml/src/ggml-common.h`): @@ -66,12 +66,12 @@ byte-products = 2 sdot for llama.cpp (`quants.c:3747-3750` etc.). Ours: per bloc surrounding work.** **Our GEMV re-uses nothing.** `kq_gemv_gen_impl` drives `emit_slice(te, sa, ghead, 0, 1, "")` -(`dasllama_gemm_gen.das:2880`) — tokCount = 1. Every decoded weight is consumed by exactly +(`dasllama_gemm_gen.das:2880`) - tokCount = 1. Every decoded weight is consumed by exactly one sdot. The alloca panel (`LLVMBuildAlloca(..., 256 * te.interleave, "iq3s.panel")`, line -2845 — 2048 bytes at mr=8) therefore buys **no reuse whatsoever in decode**; it exists only +2845 - 2048 bytes at mr=8) therefore buys **no reuse whatsoever in decode**; it exists only to separate the rolled gather loop from the unrolled dot body. The prefill/tile path is the one that benefits, and there the panel is produced once per group by -`unpack_kq_panel_grp` (`dasllama_repack.das:953`) and amortized over 4 tokens — which is +`unpack_kq_panel_grp` (`dasllama_repack.das:953`) and amortized over 4 tokens - which is exactly why every prefill tile is 1.9x-15x ahead while decode is behind. --- @@ -142,7 +142,7 @@ indeed its slowest but one. (3986, 3991) = **0.5 mem + ~6 NEON per 32w** - **the `vec_index_t` union (3941-3944)**: `idx.vec_index` is written as a vector and then read back as `idx.index[0..7]` scalars (3987-3990, 3992-3995). That is a store-forwarding - round trip through the stack — 1 vector store + 8 `ldrh` per half, **~9 mem per 32w** + round trip through the stack - 1 vector store + 8 `ldrh` per half, **~9 mem per 32w** - grid: 8 x `ldr w` + 6 lane inserts per 32w = **8 mem + 6 int** - signs: the mask1/mask2 expansion (3998-4002, 4007-4011) = **7 NEON**, plus `vmulq_s8` x2 (4004-4005, 4015-4016) = **2 NEON** @@ -155,13 +155,13 @@ its 5682 us. Two loops per superblock, both walked per row group of mr = 8: -**(A) `emit_kq_gather` (`dasllama_gemm_gen.das:1451-1486`)** — a rolled loop over the mr +**(A) `emit_kq_gather` (`dasllama_gemm_gen.das:1451-1486`)** - a rolled loop over the mr rows; each iteration emits **64 straight-line dword decodes** (`for blk in 0..7`, `for l in 0..3`, `for h in 0..1`, line 1470-1477), each decoding **4 weights of one row** and storing them with `LLVMBuildStore` into the panel at `panel_group_off` (1443-1449, 1478-1479). -**(B) the dot body inside `emit_block_iq4xs` (1573-1712)** — per block: 2 activation +**(B) the dot body inside `emit_block_iq4xs` (1573-1712)** - per block: 2 activation `v16i8` loads shared across all 8 rows (1620-1621); per (blk, qd) one `kq_sign_bytes` (1632); per (blk, j, qd) two panel `v16i8` loads (1642-1643), two `apply_sign_col` (1645-1646), two `dot_lane` (1659-1662). @@ -240,7 +240,7 @@ Each entry is tagged **emitter change** / **repack change** / **seat**, with its exposure. Zen2 is at 0.99x-1.32x on these five, so nothing here may touch the AVX2 lattice without a re-measure. -### D1 — the gather decodes one dword per scalar step and round-trips 32 B/row/block through an alloca +### D1 - the gather decodes one dword per scalar step and round-trips 32 B/row/block through an alloca **Payoff: the largest. Tag: emitter change (GEMV-only, sdot-leg-only). x86: none.** @@ -259,28 +259,28 @@ formats one grid entry is 8 consecutive weights `l*8 .. l*8+7`, but `lsr #32` to split it (`grid_pair_half`, 1324-1330). llama.cpp never splits: `vcombine_s8(vld1_s8(grid+i), vld1_s8(grid+j))` (3735) puts two -whole u64 entries — 16 consecutive weights of one row — into one q register with two loads +whole u64 entries - 16 consecutive weights of one row - into one q register with two loads and no store at all. -### D2 — the three ksigns formats synthesize the eighth sign bit with a 3-step parity chain, per group, per block, per row +### D2 - the three ksigns formats synthesize the eighth sign bit with a 3-step parity chain, per group, per block, per row **Payoff: 3.25 cyc/row/32w, ~29% of iq2_xs's time. Tag: emitter change (if D1 lands) or repack change (standalone). x86: shared code, must re-measure zen2.** `kq_sign_bytes` lines 1400-1412: `code` -> `par = code ^ (code>>4)`, `par ^= par>>2`, `par ^= par>>1`, `sb = code | ((par&1)<<7)`, then a 4-way byte pack. 13 vector ops per sign -group, 4 groups per block, on `<4 x i32>` — 53 NEON uops per (4 rows, 32 weights). +group, 4 groups per block, on `<4 x i32>` - 53 NEON uops per (4 rows, 32 weights). -llama.cpp: `vld1_s8(signs64 + code)` — one load of a pre-expanded +/-1 vector +llama.cpp: `vld1_s8(signs64 + code)` - one load of a pre-expanded +/-1 vector (`quants.c:3739-3742`, table at 3595), then one `vmulq_s8`. The parity is baked into the table. Note the comment drift here: lines 1484-1485, 1507, 1529 and 1549 all describe "the shared ksigns table" and "the shared smask table" as emitted globals, but the `*_emit_globals` -functions only create the grid global — no ksigns or smask constant is emitted, and +functions only create the grid global - no ksigns or smask constant is emitted, and `kq_sign_bytes` reads none. Those four comment fragments are stale. -### D3 — the u64 grid's 8 weights are decoded as two separate 4-weight steps +### D3 - the u64 grid's 8 weights are decoded as two separate 4-weight steps **Payoff: it is what makes D1's 8 stores 8 instead of 4, and it doubles the index work for iq2_xs. Tag: emitter change. x86: same code path, gated by the same `gather` flag.** @@ -290,7 +290,7 @@ per call. LLVM's CSE folds the two calls onto one load, but the *consumers* are independent 4-byte stores at unrelated panel offsets, so the pair can never become a single 8-byte store. -### D4 — the packed plane's 4-byte column layout costs a dword load plus shift and mask per index byte +### D4 - the packed plane's 4-byte column layout costs a dword load plus shift and mask per index byte **Payoff: ~2 cyc/row/32w on iq2_xs specifically. Tag: repack change or seat. x86: the current shape was chosen ON x86.** @@ -307,7 +307,7 @@ than two bytes"). Those figures are from the x86 development box, and the same c 1342 already admits the column read *lost* for iq3s/iq2s/iq2xxs. **The decode shape is currently one global decision made on one ISA.** It should be a seat. -### D5 — the sign application is a 5-op masked negate rather than one multiply +### D5 - the sign application is a 5-op masked negate rather than one multiply **Payoff: small on its own (~1 cyc/row/32w), large once D1 lands. Tag: emitter change.** @@ -334,26 +334,26 @@ iq2_s measures 1.00x. **Verdict: this is the fix. Adopt it.** Layout: vector `W = [ row A weights e..e+7 | row B weights e..e+7 ]`, one 16-byte register -from two 8-byte grid loads (`ldr d` into lane 0, `ld1 {v.d}[1]` into lane 1 — in LLVM, a +from two 8-byte grid loads (`ldr d` into lane 0, `ld1 {v.d}[1]` into lane 1 - in LLVM, a load plus an `insertelement` into a `<2 x i64>`). The dot then uses the **non-indexed** `sdot Vd.4S, Vn.16B, Vm.16B`, whose lane i is `Vn[4i..4i+3] . Vm[4i..4i+3]`. Feed it `X = [ x(e..e+3) | x(e+4..e+7) | x(e..e+3) | -x(e+4..e+7) ]`, i.e. the 8 activation bytes replicated — exactly `ld1r {v.2d}, [xp]`, one +x(e+4..e+7) ]`, i.e. the 8 activation bytes replicated - exactly `ld1r {v.2d}, [xp]`, one instruction, no lane-splat shuffle at all. Then: - lane 0 = row A . x(e..e+3), lane 1 = row A . x(e+4..e+7) - lane 2 = row B . x(e..e+3), lane 3 = row B . x(e+4..e+7) Accumulate all 4 element-groups of a block into **one** `<4 x i32>` per row pair; at block -end, one `addp` of two row-pair accumulators yields `[rowA, rowB, rowC, rowD]` — **the exact +end, one `addp` of two row-pair accumulators yields `[rowA, rowB, rowC, rowD]` - **the exact `<4 x i32>` shape `a[]` already has** (line 1659-1662), so the entire scale/fold epilogue (1683-1711) is untouched. `pairwise_add_i32` (909-917) already emits that `addp`. Products per sdot: 2 rows x 8 weights = 16, identical to today's 4 x 4. **No dot regression.** Same intrinsic as today (`llvm.aarch64.neon.sdot`, `<4 x i32>`/`<16 x i8>`, declared at -1911-1918) — only the operand construction changes, so no new decline rail. +1911-1918) - only the operand construction changes, so no new decline rail. Projected budget per **2 rows x 32 weights** for iq2_xs, keeping the panel: @@ -380,13 +380,13 @@ combined dot body (~900) plus gather body. Worth trying second. Notes and caveats: - **The +/-1 sign table replaces D2 entirely.** Under this layout the sign vector for a row - pair is two 8-byte loads from a `keven_signs`-equivalent global indexed by the 7-bit code — + pair is two 8-byte loads from a `keven_signs`-equivalent global indexed by the 7-bit code - no parity chain, no plane change, no repack. D2 becomes unnecessary if (a) lands. - **For iq2_s / iq3_s** (signs are plane *bits*, not codes) emit a 256-entry x 8-byte +/-1 table indexed by the raw sign byte (2 KB private constant, same shape as the grid globals at 1273-1291) so all five formats share one sign path. - **For the u32-grid formats** (iq3_xxs, iq3_s) a row's 8 weights are two grid entries, so - building the half costs 2 loads (`ldr s` + `ld1 {v.s}[1]`) instead of 1 — 16 grid loads per + building the half costs 2 loads (`ldr s` + `ld1 {v.s}[1]`) instead of 1 - 16 grid loads per (2 rows, 32w) instead of 8. Projected ~5 cyc/row/32w, still ~2.3x better than today and ahead of llama.cpp, which pays the same 8 u32 loads per row plus 6 lane inserts (`quants.c:3896-3899`, `ggml-cpu-impl.h:84`). @@ -404,9 +404,9 @@ Notes and caveats: **Verdict: impossible for the grid. Not worth an experiment.** -NEON `TBL`/`TBX` index into 1 to 4 *consecutive V registers* — a 16, 32, 48 or 64-byte +NEON `TBL`/`TBX` index into 1 to 4 *consecutive V registers* - a 16, 32, 48 or 64-byte table, with out-of-range indices producing zero. The smallest grid here is `iq3xxs_grid`, -256 x `uint32_t` = **1024 bytes** (`ggml-common.h:1017`) — 64 V registers, 16x the ISA +256 x `uint32_t` = **1024 bytes** (`ggml-common.h:1017`) - 64 V registers, 16x the ISA maximum. `iq2xxs_grid` is 2048 bytes (`ggml-common.h:560`). There is no NEON gather instruction of any kind, and the M1 has no SVE (profile features string: `neon;dotprod;fullfp16;lse`). So the grid must be read with scalar loads on this ISA, and @@ -414,9 +414,9 @@ llama.cpp does exactly that (`vld1_s8`/`ldr w` per entry) for the same reason. Where `tbl` *is* the right tool, and where both sides already use it: -- the 16-entry nibble codebook for iq4_nl/iq4_xs — we already emit `llvm.aarch64.neon.tbl1` +- the 16-entry nibble codebook for iq4_nl/iq4_xs - we already emit `llvm.aarch64.neon.tbl1` (`dasllama_gemm_gen.das:1929-1936`, `lut_lookup` 632-642) -- broadcasting a sign byte across its four lanes — `apply_sign_col`'s constant +- broadcasting a sign byte across its four lanes - `apply_sign_col`'s constant `shufflevector` (1421-1422) lowers to `tbl`, and llama.cpp's `mask1` shuffle (`quants.c:3822`, `4000`) is the same instruction @@ -455,9 +455,9 @@ Two ways to kill it: bytes, and the scale already lives in the scale plane (`IQ2XXS_SSB` comment, `dasllama_kqformat.das:118`), so 4 sign bytes fit in the same 4 bytes. **Size-neutral** (`IQ2XXS_QSB` stays 64, line 117). - - **iq3_xxs**: identical — 8 blocks x 4 aux bytes -> 8 x 4 sign bytes. + - **iq3_xxs**: identical - 8 blocks x 4 aux bytes -> 8 x 4 sign bytes. **Size-neutral** (`IQ3XXS_QSB` stays 96, line 111). - - **iq2_xs**: the 9-bit index does not fit a byte. Restructure to iq2_s's shape — + - **iq2_xs**: the 9-bit index does not fit a byte. Restructure to iq2_s's shape - 32 index low bytes + 4 high-bit bytes + 32 sign bytes = 68 B against today's 64 (`IQ2XS_QSB`, line 116). **+6%.** Projected under this alone: iq2_xs 6375 -> ~4520 (0.72x), iq2_xxs 6063 -> ~4390 (0.78x), @@ -466,9 +466,9 @@ Two ways to kill it: **x86 caution for option 2**: it changes the on-disk plane, the repack, the tile unpack (`dasllama_repack.das:874-949`) and both ISAs. The +6% on iq2_xs is affordable on - bandwidth grounds — the decode moves 16 MB per call in 6375 us = 2.5 GB/s against M1 + bandwidth grounds - the decode moves 16 MB per call in 6375 us = 2.5 GB/s against M1 Max's tens of GB/s single-core, so both implementations are ~15-25x off the memory roof - and firmly compute-bound — but zen2's iq2_xs must be re-measured before it lands. iq2_xxs + and firmly compute-bound - but zen2's iq2_xs must be re-measured before it lands. iq2_xxs and iq3_xxs being size-neutral carry no bandwidth risk on either ISA. --- From b61b88decc38d0ef1653e2c225387f422a65e14e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:13:57 -0700 Subject: [PATCH 016/123] llvm_tune: the x86-amx CPU class (AMX-INT8 + AMX-TILE over the 512-bit VNNI gate), first in the x86 chain An AMX box with no AMX profile adopts the vnni512 profile and races only the seats that class could not answer. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLVM/daslib/llvm_tune.das | 4 ++-- skills/tune.md | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index 476505a7c7..1818680066 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -1256,7 +1256,7 @@ def tune_cpu_class() : string { let arch = get_architecture_name() if (arch == "x86_64") { if (fallback_feature_ok("avx512vnni") && fallback_feature_ok("avx512bw")) { - return "x86-vnni512" + return (fallback_feature_ok("amx-int8") && fallback_feature_ok("amx-tile")) ? "x86-amx" : "x86-vnni512" } if (fallback_feature_ok("avxvnni")) { return "x86-vnni256" @@ -1273,7 +1273,7 @@ def tune_cpu_class() : string { //! profile adopts the nearest lower class and races only what that class could not. def tune_class_chain() : array { let ladder = (get_architecture_name() == "x86_64" - ? "x86-vnni512;x86-vnni256;x86-avx2;x86-base" + ? "x86-amx;x86-vnni512;x86-vnni256;x86-avx2;x86-base" : "arm-i8mm;arm-neon") let own = tune_cpu_class() var chain : array diff --git a/skills/tune.md b/skills/tune.md index cb5a7b0fcf..80aa9375a9 100644 --- a/skills/tune.md +++ b/skills/tune.md @@ -174,8 +174,9 @@ fingerprint (and any `version_of=` pin value). Runtime knobs, race tables and box identity never travel - all three are properties of the box that measured them, not of its class. -The class names follow the features `requires=` can gate: `x86-vnni512` / -`x86-vnni256` / `x86-avx2` / `x86-base`, `arm-i8mm` / `arm-neon`; any other +The class names follow the features `requires=` can gate: `x86-amx` (AMX-INT8 + +AMX-TILE over the 512-bit VNNI gate) / `x86-vnni512` / `x86-vnni256` / `x86-avx2` / +`x86-base`, `arm-i8mm` / `arm-neon`; any other architecture gets `-`, a key no shipped profile matches, so such a box always races (`tune_cpu_class()` computes this box's, `tune_class_chain()` its adoption ladder). An untuned auto/restart start walks the ladder from the box's own From 220fcec7dfb3ceb3fe9d867fb4df26e10de6b89c Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:15:20 -0700 Subject: [PATCH 017/123] followup_general 64: Qwen 3.8 thinking control (reasoning_effort low/medium/xhigh, the budget lever, the token penalty); 63 amended with the rollback cost Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/followup_general.md | 20 + plans/qwen38_thinking_control.md | 597 +++++++++++++++++++++++++++ 2 files changed, 617 insertions(+) create mode 100644 plans/qwen38_thinking_control.md diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 6c653bb70d..822d7796bc 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -805,3 +805,23 @@ counter, so the rewind is "n_past back k, drop k cache rows" plus a phrase table and a ban list in the sampler; expose it as a server/CLI knob. Done = the knob, a test that a forced ", but wait" stream rewinds and continues, and a before/after token count on a 27B reasoning prompt. + 2026-09-01 research (`plans/qwen38_thinking_control.md`): the phrase-level rollback is what + Antislop does and costs 69-96% of throughput; the cheap form that the two 2026 papers measure is a + SINGLE-TOKEN logit penalty on the reconsideration tokens ("wait", "but", "alternatively") at + 12-51% shorter traces with equal or better accuracy - build that first, the rollback only if the + single-token penalty measures short. + +64. **Qwen 3.8 thinking control: expose quick / normal / high (Boris, 2026-09-01).** The model's one + first-party knob is `reasoning_effort` with exactly three legal values - `xhigh` (default), + `medium`, `low` - which the chat template turns into one system-prompt sentence (`medium` injects + nothing; anything else, OpenAI's usual `high` included, raises in the template). No budget exists in + the model; `` and `` are single vocab tokens (248068 / 248069), so an early stop is a + one-token force. What our stack lacks (`plans/qwen38_thinking_control.md`): a `dasllama_arch_qwen38.das` + (the GGUF spec exists), the template's `` prefill in `chatml_chat`, and the server drops + `reasoning_effort`. The three levers, in order: (1) map quick/normal/high -> low/medium/xhigh in the + template's effort sentence and accept `reasoning_effort` in the server (translate `high` to `xhigh` + instead of raising); (2) a reasoning token budget in `SamplingParams` that forces `` - the + counter must see the PROMPT-side `` (the template opens the block), so prefill tokens pass + through the sampler's state; (3) the single-token reconsideration penalty of entry 63. Nobody + publishes what the effort levels cost; measure our three rungs on the 27B before naming them. + diff --git a/plans/qwen38_thinking_control.md b/plans/qwen38_thinking_control.md new file mode 100644 index 0000000000..fbed5a9692 --- /dev/null +++ b/plans/qwen38_thinking_control.md @@ -0,0 +1,597 @@ +# How much Qwen 3.8 thinks, and who controls it + +Research memo. Written 2026-09-01. Read-only research; nothing in any checkout was modified. + +Evidence base: +- Qwen's own files, downloaded fresh from Hugging Face into `/tmp` during this session: + `Qwen/Qwen3.8-27B` `chat_template.jinja`, `README.md`, `tokenizer_config.json`, + `generation_config.json`, and `Qwen/Qwen3.8-Flash-Next` `chat_template.jinja`, `README.md`. +- The local llama.cpp checkout `D:\Work\llama.cpp`, at HEAD `6c84c7d5d` ("model: add + Qwen3.8-Flash-Next (qwen4exp) (#27742)"). All `llama.cpp` line numbers below are that commit. +- The local daslang checkout `D:\Work\daScript` (module `modules/dasLLAMA`, server + `utils/dasllama-server`). +- Public docs and papers, cited inline. + +--- + +## Summary in one paragraph + +Qwen 3.8 has exactly one first-party knob for the *amount* of thinking, and it is a prompt +knob, not a budget: `reasoning_effort`, with three legal values `xhigh` (the default), `medium`, +and `low`. The chat template turns that value into one English sentence which it prepends to +the system turn; the model was trained to respond to that sentence. There is a second knob, +`enable_thinking`, but that is on/off, not an amount. Qwen ships **no** token budget, **no** +`thinking_budget` template variable, **no** budget-forcing text, and **no** `/think` `/no_think` +soft switch in the 3.8 template. Everything that actually *caps* the thinking length is bolted +on by the serving stack: llama.cpp has a full reasoning-budget sampler, vLLM has +`thinking_token_budget`, SGLang has a `thinking_budget` that is reported broken. Both +Qwen3.8-27B and Qwen3.8-Flash-Next (`qwen4exp`) ship a byte-identical chat template, so +everything here applies to both without change. + +--- + +## (a) The mechanisms that actually exist + +### A1. `enable_thinking` — a trained-in on/off switch, not an amount + +Template spelling (`Qwen/Qwen3.8-27B/chat_template.jinja`, lines 163-170): + +```jinja +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} +``` + +Two things to notice. + +1. With thinking **on** (the default), the template itself writes the opening `\n` into + the generation prompt. The model never emits ``; it starts already inside the thought + block and only has to emit `` to leave it. This matters for any budget mechanism — + see A6 and B3. +2. With thinking **off**, the template writes an already-closed empty block + `\n\n\n\n`. This is the same "prefill an empty closed block" trick Qwen3 used. + +API spelling, per the model card (`README.md` lines 296, 459): + +```python +extra_body={"chat_template_kwargs": {"enable_thinking": False}} +``` + +The card adds (line 466): on Qwen Cloud the field is top-level `"enable_thinking": False`, not +wrapped in `chat_template_kwargs`. + +Source: and the raw template at +. + +### A2. `reasoning_effort` — the amount knob. It is a system-prompt sentence. + +This is the only first-party control over *how much* the model thinks. Template lines 45-56, +verbatim: + +```jinja +{%- set reasoning_instructions = '' %} +{%- if enable_thinking is undefined or enable_thinking is true %} + {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %} + {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %} + {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }} + {%- endif %} + {%- if resolved_reasoning_effort == 'xhigh' %} + {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %} + {%- elif resolved_reasoning_effort == 'low' %} + {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %} + {%- endif %} +{%- endif %} +``` + +Five facts follow directly from that code, and they are the most useful things in this memo: + +1. **The whole mechanism is one sentence of English prepended to the system turn.** There is no + special token, no logits change, no counter. The `reasoning_instructions` string is spliced + in ahead of the user's system content (template lines 58-86) — and if the request carries no + system message at all, the template *creates* a system turn holding only that sentence + (lines 81-85). +2. **`medium` injects nothing.** Only `xhigh` and `low` set the string; `medium` leaves it + empty. So a request with `reasoning_effort="medium"` renders to a prompt that is + byte-identical to a request with thinking on and no effort field. `medium` is the neutral + baseline; `xhigh` and `low` are the two deviations from it. +3. **The default is the expensive end.** `reasoning_effort|default('xhigh')` — omitting the + field gets you the "think carefully, validate assumptions, consider alternatives" sentence. +4. **Only three values are legal.** `low`, `medium`, `xhigh`. Anything else — including + OpenAI's own standard `"high"` — makes the template raise, and the request fails. This is a + sharp trap for an OpenAI-compatible front end, because `high` is the value most clients send. +5. Because it is only a prompt sentence, it is **advisory**. The model was post-trained to obey + it, but nothing enforces it, and there is no upper bound on the trace length at any level. + +Model card wording (`README.md` lines 258-262): + +> Qwen3.8 comes with official support for `reasoning_effort`, which can be used to adjust +> reasoning depth and control cost: +> - `xhigh` (default): for complex tasks demanding thorough analysis +> - `medium`: balancing accuracy and speed +> - `low`: efficient reasoning optimizing for speed and cost + +The card also carries an explicit warning against assuming lower effort is always faster +end-to-end (`README.md` line 267): + +> In multi-turn agentic tasks, lower reasoning effort does not always reduce overall task +> completion time. Although it may produce faster per-turn responses, it can also lead to +> insufficient analysis, more failures, and repeated retries, which may increase total latency +> and token consumption. + +On the wire it is a **top-level OpenAI field**, not a template kwarg, in Qwen's own example +(`README.md` line 300): + +```python +completion = client.chat.completions.create( + model="Qwen/Qwen3.8-27B", + messages=messages, + extra_body={"chat_template_kwargs": {"enable_thinking": True, "preserve_thinking": True}}, + reasoning_effort="xhigh", # xhigh by default; supported levels are xhigh, medium, and low + ... +) +``` + +vLLM's own recipe page for this model agrees on all of the above: +. + +### A3. `preserve_thinking` — history retention, which is an amount knob at second hand + +Template lines 111-119: + +```jinja +{%- set reasoning_content = '' %} +{%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} +{%- endif %} +{%- set reasoning_content = reasoning_content|trim %} +{%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} +{%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} +{%- endif %} +``` + +Defaults to true. Note the retention is driven by a `reasoning_content` field on each historical +assistant message — so a client that strips reasoning before storing history silently gets +`preserve_thinking` behaviour with empty thoughts (an empty `\n\n` per turn), +which is neither of the two documented modes. The card claims retention improves KV-cache reuse +and agent consistency (`README.md` line 472). Turning it off shortens the prompt but the card +warns it costs decision consistency in agent loops. + +Note the `or loop.index0 > ns.last_query_index` clause: thinking is *always* kept for assistant +messages after the last real user query, i.e. within the current tool-call chain, regardless of +the flag. + +### A4. What Qwen does *not* ship + +Checked directly against the downloaded files, all negative: + +- **No `thinking_budget` / `reasoning_budget` template variable.** `grep -n "budget" chat_template.jinja` + returns nothing. +- **No budget-forcing text.** The "Considering the limited time by the user, I have to give the + solution based on the thinking directly now." style closer is **not** Qwen's; it is a serving + convention (see A6). +- **No `/think` or `/no_think` soft switch.** Those were Qwen3-generation features documented on + the Qwen3 model cards (). `grep -n "no_think\|/think"` + finds no handling in the 3.8 template and no mention in either 3.8 model card. Whether the 3.8 + weights still react to those strings as plain prompt text is untested — see (c). +- **No special reasoning-length token.** Nothing in the vocab named for effort or budget. + +### A5. Token-level facts you need for any sampler-side work + +From `Qwen/Qwen3.8-27B/tokenizer_config.json`, `added_tokens_decoder`: + +| token | id | `special` flag | +|---|---|---| +| `<|im_start|>` | 248045 | true | +| `<|im_end|>` | 248046 | true | +| `` | 248058 | false | +| `` | 248059 | false | +| `` | **248068** | false | +| `` | **248069** | false | + +`` and `` are **single vocabulary entries**. Forcing an early end of thinking is +therefore a one-token force, not a multi-token string force. (`special: false` here only means +they are not stripped by `skip_special_tokens`; they are still atomic added tokens.) + +`generation_config.json`: `temperature 1.0`, `top_k 20`, `top_p 0.95`, `eos_token_id [248046, 248044]`. + +Recommended sampling per the card (`README.md` lines 252-253) — worth quoting because it differs +between modes: +- Thinking: `temperature=1.0, top_p=0.95, top_k=20, min_p=0.0, presence_penalty=0.0, repetition_penalty=1.0` +- Non-thinking: `temperature=0.7, top_p=0.80, top_k=20, min_p=0.0, presence_penalty=1.5, repetition_penalty=1.0` + +And the agentic sizing guidance (`README.md` lines 507-510), which is the closest Qwen comes to a +budget: *"Reasoning Content: Set the maximum output length to 262,144 tokens. Final Response: Set +the maximum output length to 131,072 tokens."* That is a recommendation to size two separate +caps, not a mechanism. + +### A5b. Qwen3.8-Flash-Next / `qwen4exp` is the same story + +`diff` of the two downloaded templates after trailing-whitespace normalisation: **identical**. +The Flash-Next card says the same thing in prose (`README.md` line 350): *"Qwen3.8-Flash-Next +supports controlling thinking behavior via `enable_thinking`, `preserve_thinking`, and +`reasoning_effort`."* Same three effort levels, same `xhigh` default (line 384). + +The llama.cpp commit `6c84c7d5d` that added `qwen4exp` is purely architectural — hyper-connections, +gated delta net, MoE, PLE n-gram embeddings, UINT64 GGUF arrays. It touches no chat, template, or +reasoning code. `grep -rn "xhigh"` across the whole llama.cpp tree hits exactly one line, the help +text of `--reasoning-effort` (`common/arg.cpp:3711`). There is no Qwen3.8 template fixture under +`models/templates/` (newest Qwen there is `Qwen3.5-4B.jinja`) and no Qwen3.8 test in +`tests/test-chat.cpp`. + +### A6. What serving stacks bolt on + +#### llama.cpp — a real reasoning-budget sampler (the most complete implementation seen) + +`D:\Work\llama.cpp`, HEAD `6c84c7d5d`. + +**The sampler.** `common/reasoning-budget.h:10-16` declares a five-state machine: + +``` +IDLE -> COUNTING -> WAITING_UTF8 -> FORCING -> DONE +``` + +- `IDLE`: passthrough, watching for the start sequence. +- `COUNTING`: decrement per token, watching for a natural end sequence. +- `WAITING_UTF8`: budget spent, but let the current multi-byte character finish. +- `FORCING`: emit the forced sequence token by token. `common/reasoning-budget.cpp:166-186` is + the whole enforcement — every logit except the one forced token is set to `-INFINITY`. +- `DONE`: passthrough. It **re-arms** on a new start tag (`reasoning-budget.cpp:147-161`), + because some models open several `` blocks in one reply. + +**The forced sequence** is `reasoning_budget_message` tokens followed by the first end tag +(`tools/server/server-schema.cpp:415-427`). That is exactly the "Considering the limited time…" +pattern: you supply the sentence, the server prepends it to `` and forces the lot. + +**CLI flags** (`common/arg.cpp`): + +| line | flag | meaning | +|---|---|---| +| 3680 | `--reasoning-format none\|deepseek\|deepseek-legacy` | where thoughts land in the response | +| 3691 | `-rea, --reasoning on\|off\|auto` | sets `enable_thinking` template kwarg | +| 3709 | `--reasoning-effort LEVEL` | passes the string straight into the jinja context | +| 3721 | `--reasoning-budget N` | `-1` unlimited, `0` immediate end, `N` token budget | +| 3729 | `--reasoning-budget-message MESSAGE` | text injected before the end tag on exhaustion | +| 3736 | `--reasoning-preserve` / `--no-reasoning-preserve` | sets the `preserve_reasoning` kwarg | + +Environment aliases exist for each (`LLAMA_ARG_REASONING`, `LLAMA_ARG_REASONING_EFFORT`, +`LLAMA_ARG_THINK_BUDGET`, `LLAMA_ARG_THINK_BUDGET_MESSAGE`). + +**Per-request HTTP fields** (`tools/server/server-common.cpp:1313-1378`): + +- `chat_template_kwargs: {"enable_thinking": bool, ...}` — merged over the server defaults. +- `reasoning_effort: ""` — the OpenAI field. `"none"` is special-cased to + `enable_thinking = false` (line 1326-1328); any other non-empty string is forwarded as the + template kwarg (line 1330). +- `reasoning_budget_tokens`, alias `thinking_budget_tokens` (line 1365-1366), falling back to the + server default. +- `reasoning_budget_message` (line 1375). +- `reasoning_control: bool` (line 1376) — arms the sampler so it can be forced later at runtime. + +**Runtime early-stop.** `POST /v1/chat/completions/control` with +`{"id": , "action": "reasoning_end"}` forces the in-flight completion out of its +thought block mid-stream (`tools/server/server-context.cpp:2431-2447`, documented at +`tools/server/README.md:1454`). It requires `reasoning_control: true` on the original request. +This is a "stop thinking, answer now" button for a UI. + +**Two llama.cpp findings that bear on Qwen 3.8 specifically:** + +1. **The budget arms correctly despite the template pre-filling ``.** Recall from A1 that + the model never emits `` — the template does. llama.cpp handles this: the generation + prompt is re-tokenized into `prefill_tokens` (`common/sampling.cpp:279-292`) and those tokens + are fed through `llama_sampler_accept` on the budget sampler at construction + (`common/sampling.cpp:318-322`), so the `` in the prompt moves the state machine + `IDLE -> COUNTING`. Any stack that rolls its own budget must do the same or the counter never + starts. +2. **`--reasoning-preserve` does not work on Qwen 3.8.** It writes the template kwarg + `preserve_reasoning` (`common/arg.cpp:3742,3745`); the Qwen 3.8 template reads + `preserve_thinking` (template line 116). The names differ, so the flag is inert here. The + working spelling is `chat_template_kwargs: {"preserve_thinking": false}`. + +**How llama.cpp routes the Qwen 3.8 template.** There is no Qwen3.8 branch. The template +contains ``, `` in the +template source (`common/chat.cpp:1172`), which succeeds, so +(`common/chat.cpp:1183-1186`): + +```cpp +data.thinking_start_tag = ""; +data.thinking_end_tags = { "", "" }; +``` + +Both of those flow into the budget sampler. So `--reasoning-budget` works on Qwen 3.8 out of the +box, provided `--jinja` is on. Whether `reasoning_effort` is offered to the client is decided by +a capability probe that renders the template with `reasoning_effort = "low"` and checks whether +the variable was read (`common/jinja/caps.cpp:526-532`); Qwen 3.8 reads it, so it probes as +supported. + +#### vLLM + +- Sampling parameter `thinking_token_budget` — per-request reasoning token limit. +- `--reasoning-parser qwen3` is required for Qwen 3.8, because the template opens every assistant + turn with `` and without the parser the whole thought block lands in `content` + (). +- `--reasoning-config '{"reasoning_start_str": "", "reasoning_end_str": "I have to give the + solution based on the reasoning directly now."}'` — this is where the "limited time" + style closer comes from. The docs say putting transitional language in `reasoning_end_str` + makes the termination "more natural". +- `--default-chat-template-kwargs '{"enable_thinking": false}'` for a server-wide default; + per-request `chat_template_kwargs` always wins. +- The docs list Qwen3, DeepSeek and Nemotron3 as the families supporting a thinking budget. +- Source: . + +#### SGLang + +Exposes `chat_template_kwargs` and `separate_reasoning: true` (which populates +`reasoning_content`), plus a `thinking_budget`. The budget is reported **not enforced** on this +model generation: sgl-project/sglang issue #25536, "thinking_budget not enforced for Qwen3.6 — +reasoning consumes all max_tokens", where `thinking_budget: 200` still produced roughly 1400 +reasoning tokens and left nothing for the answer +(). Treat SGLang's budget as unreliable. + +#### Ollama + +Reported in the model discussions to replace the template with a generic one, which silently +disables `reasoning_effort` entirely +(). + +### A7. Independent write-ups on quantization and on banning reconsideration phrases + +This is directly relevant to us because we run this model quantized. + +**Quantization makes reasoning traces longer.** Two 2026 papers: + +- *"Quantized Reasoning Models Think They Need to Think Longer, but They Do Not"* + (). Abstract, verbatim in the load-bearing part: *"aggressive + PTQ reduces accuracy while increasing chain-of-thought (CoT) length… in up to 52% of the + quantized models' failures, models reach the right answer in intermediate reasoning steps but do + not output it as a final answer… Positions with high KL divergence correlate strongly with high + next-token entropy, and at these positions quantized models disproportionately sample + overthinking markers such as "wait", "but", and "alternatively". We show that simply introducing + a training-free logit penalty on a curated set of overthinking markers can reduce CoT length by + 12--23% while preserving or improving accuracy across 5 models (1.5B-32B parameters), 3 + quantization methods, and 5 benchmarks… Overthinking errors produced by quantized models are + particularly reduced by up to 58%."* +- *"Quantization Inflates Reasoning: Token Inflation as a Hidden Cost of Low-Bit Reasoning Models"* + (). INT4/INT3 can preserve accuracy while inflating reasoning + token count enough to cancel the per-token speedup, with more intermediate steps and more + semantic repetition in the trace. + +The practical reading: a Q4_K_M Qwen 3.8 is expected to think *longer* than the bf16 model at the +same `reasoning_effort`, and the extra length is disproportionately made of "wait" / "but" / +"alternatively" continuations sampled at high-entropy positions. A logit penalty on those markers +is the paper's own remedy and it is cheap. + +**Banning reconsideration phrases outright.** *"Wait, We Don't Need to 'Wait'! Removing Thinking +Tokens Improves Reasoning Efficiency"* (NoWait), — suppresses +explicit self-reflection tokens (`Wait`, `Hmm`) at decode time and reports a 27%-51% shorter +chain of thought across five R1-style model series and ten benchmarks with utility preserved. + +**Rollback and resampling** is the Antislop sampler +(, summary at +). Unlike a logit ban it works on +*multi-token phrases*: it keeps the inference trace, and when a banned pattern completes it +backtracks to the token where the pattern began, downweights the offending continuation, and +resamples. The suppression is soft — `p_new = p_old * 10^(-10s)` with ban strength `s` in `[0,1]`, +`s=1` being a hard ban. It scales to 8000+ patterns (direct token banning caps out near 2000) and +accepts regexes. The cost is real: 69%-96% output slowdown in the bad cases. This is the only +mechanism in the set that can ban a phrase like `", but wait"` precisely — a plain logit ban on +the `wait` token cannot distinguish `", but wait"` from a legitimate `wait`. + +For comparison, llama.cpp itself has **no** backtracking sampler (`grep -rni "antislop\|backtrack"` +over `src/ common/ tools/` finds only unrelated tokenizer and unicode code). It has DRY +(`common/sampling.cpp:353`) and `logit_bias`, which are enough for the NoWait-style single-token +ban but not for phrase-level rollback. + +--- + +## (b) What we could expose as "quick" / "normal" / "high" + +Our stack, for reference: chat rendering is `modules/dasLLAMA/dasllama/dasllama_chat.das` over +per-family `ChatTemplate` data declared in `dasllama_arch_*.das` / +`dasllama_common.das:1120-1150`; sampling is +`modules/dasLLAMA/dasllama/dasllama_sampling.das:23-32` (`SamplingParams`: temp, top_k, top_p, +min_p, repetition/presence/frequency penalties — no logit bias, no forced tokens, no phrase ban); +the OpenAI server is `utils/dasllama-server/openai_server.das`. The design overview is +`modules/dasLLAMA/THINKING.md`. We already carry a Qwen3.8-27B GGUF spec +(`modules/dasLLAMA/performance/model_specs.das:346-351`, `Qwen3.8-27B-UD-Q4_K_M.gguf`, +`serve_ctx = 262144`) but there is **no** `dasllama_arch_qwen38.das` yet, and `chatml_chat` +(`dasllama_common.das:4639-4653`) does not prefill `` on the assistant open, which the +Qwen 3.8 template does. + +The answer to the question is **yes**, and there are three independent levers. I would build them +in this order. + +### B1. Ship `reasoning_effort` as the primary knob (cheap, first-party, no sampler work) + +Map the three rungs straight onto Qwen's three levels: + +| our rung | `reasoning_effort` | what the model sees | +|---|---|---| +| quick | `low` | the "keep your thinking brief and focused" sentence | +| normal | `medium` | no sentence at all (the neutral prompt) | +| high | `xhigh` | the "think carefully, validate key assumptions" sentence | + +Concrete steps: + +1. **Add an effort field to `ChatTemplate`.** The cleanest shape given our existing data-driven + design is a three-slot string set — e.g. `effort_low : string`, `effort_high : string`, with + empty meaning "inject nothing", which is literally how Qwen encodes `medium`. Render it into + the system turn ahead of the user's system content, matching template lines 58-86, including + the case where there is no user system prompt and we must synthesize a system turn holding only + the sentence. +2. **Add a `reasoning_effort` field to the server.** Add `"reasoning_effort"` to `CHAT_FIELDS` + (`utils/dasllama-server/openai_server.das:1556-1559`) — today it is silently warned about as an + unsupported field. Also accept it inside `chat_template_kwargs` (the loop at line 2521-2527 + currently warns on every key but `enable_thinking`). +3. **Decide the out-of-range policy, and do not copy Qwen's.** Qwen's template hard-fails on + `"high"`, which is the single most common value an OpenAI client sends. We should map + `high` -> `xhigh` and `minimal` -> `low`, accept `none` -> thinking off (llama.cpp's rule, + `server-common.cpp:1326`), and reject the rest with a 400 naming the legal set. This is a + deliberate divergence from llama.cpp, which passes the string through and lets the template + blow up. +4. **Keep it a per-family declaration, not a Qwen special case** — some families have no effort + sentence at all and should render nothing regardless of the request. + +Caveat to state in the UI: this is a *request*, not a cap. It has no worst case. + +### B2. Add a hard token budget in the sampler (the only real cap) + +Because `` and `` are single tokens (A5), our version is much simpler than +llama.cpp's general string-sequence machine. + +Concrete steps: + +1. **Add to `SamplingParams`:** `reason_budget : int64` (`-1` off, `0` = end immediately, + `N` = token cap), plus the forced-token list and the optional closer message. +2. **Count inside the block.** The generation loop knows whether it is inside the thought span — + `dasllama_chat.das` already has the reply-side matcher (`ThinkStream` / + `make_think_stream_`/`think_feed_`, `dasllama_chat.das:1054-1057`) for exactly this. Arm the + counter when the block opens. **Critical**: with the Qwen 3.8 shape the block is opened by the + *prompt*, not by a sampled token, so the counter has to be armed at prompt-render time when + `think_open` was prefilled — this is the same bug llama.cpp avoids by feeding the prefill + tokens through the sampler (`common/sampling.cpp:318-322`). +3. **Force on exhaustion.** Set every logit but the forced token to `-inf` for as many steps as + the forced sequence is long. If a closer message is configured, force its tokens first and + `` (248069) last — this is what makes the transition read naturally rather than as an + abrupt cut. Mirror llama.cpp's UTF-8 grace state (`reasoning-budget.h:14`) so we do not sever + a multi-byte character. +4. **Re-arm on a second ``** (`reasoning-budget.cpp:147-161`) — the model can open more + than one block per reply. +5. **Server field:** accept `reasoning_budget_tokens` with alias `thinking_budget_tokens` + (llama.cpp's spelling, `server-common.cpp:1365-1366`) plus `reasoning_budget_message`. + +Suggested rung defaults, to be measured, not assumed: quick 512-1024, normal 4096-8192, high +unlimited. The community reports 22k reasoning tokens on a single SVG prompt at `xhigh` and +~60k per turn in agent use, so "unlimited" for the high rung needs a ceiling from `max_tokens` +regardless (). + +### B3. A "stop thinking now" button (small, high perceived value) + +llama.cpp's `POST /v1/chat/completions/control` with `action: "reasoning_end"` +(`server-context.cpp:2431-2447`) is about thirty lines of work on top of B2: arm the budget +machinery for the request, then let an out-of-band request drive the state machine straight to +FORCING. For an interactive UI watching a model think for four minutes this is worth more than +either of the above. + +### B4. Optional, quantization-motivated: an overthinking-marker penalty + +We serve Q4_K_M, and A7 says that specifically inflates traces with `wait` / `but` / +`alternatively`. A single-token logit penalty is a small addition to `SamplingParams` (we have +no `logit_bias` at all today) and the paper reports 12-23% shorter traces at equal or better +accuracy. Reasonable shape: a per-rung penalty, off at `high`, mild at `normal`, strong at +`quick`. + +Phrase-level bans (`", but wait"`) need rollback and resampling, which we do not have and which +costs 69-96% throughput in the reference implementation. **Do not build that.** The single-token +penalty gets most of the benefit at no throughput cost. + +### B5. Prerequisite work regardless of which rungs we ship + +- **`dasllama_arch_qwen38.das` does not exist.** Qwen 3.8 is ChatML-shaped so `chatml_chat` is + the right base, but three things differ from our Qwen3/3.5 wiring: (i) the generation prompt + must prefill `\n` when thinking is on — our `chatml_chat` does not + (`dasllama_common.das:4642-4643`), and `ChatTemplate` already has the field for it, + `assistant_open_think`, added for gemma-4 (`dasllama_common.das:1135`); (ii) the tool block is + Qwen3-Coder XML (``), **not** the Hermes JSON our + `dasllama_arch_qwen35.das:44` installs; (iii) the effort sentence from B1. +- **History retention conflicts with Qwen's default.** `THINKING.md` records that our `respond_` + stores history **reasoning-stripped**. Qwen 3.8 defaults to `preserve_thinking = true` and + renders `\n{reasoning_content}\n` for every historical assistant turn. Stripping + leaves us rendering empty thought blocks — neither documented mode. We should either carry + `reasoning_content` in history and honour `preserve_thinking`, or render the non-preserving + branch (template line 119) deliberately. Right now we do neither on purpose. + +--- + +## (c) What is uncertain or contradictory + +1. **Nobody publishes what the effort levels actually cost.** Qwen gives three adjectives. The + only numbers found are community anecdotes and they disagree wildly: one report has `medium` + cutting time by about a third with no measurable quality loss, another has `medium` moving + thinking tokens only from 106 to 91 on a short prompt while still cutting latency by a third, + a third has `xhigh` at 22,276 reasoning tokens and 21 minutes on one prompt, another claims + ~60k tokens per agent turn + (, + ). These are different prompts on + different hardware. **We should measure our own three rungs before publishing them.** + +2. **Qwen contradicts itself on whether lower effort is cheaper.** The card promotes `low` as + "optimizing for speed and cost" and then warns that in multi-turn agent work lower effort can + *increase* total latency and token consumption through retries (`README.md:267`). Both can be + true; it means a per-turn benchmark will overstate the benefit of the quick rung for agents. + +3. **`medium` being prompt-identical to "no instruction" is my reading of the template, not a + documented claim.** It follows directly from lines 45-56 (the string is only set for `xhigh` + and `low`), but Qwen never says it. It implies the model's *trained* default behaviour — what + it does with no effort sentence — is the `medium` behaviour, and that `xhigh` being the + template default means the shipped default is deliberately above the trained baseline. That + inference is untested. + +4. **Whether `/think` and `/no_think` still work is untested.** They are absent from the 3.8 + template and both 3.8 cards. The 3.8 weights may still react to them as prompt text, since + they were trained into earlier Qwen generations, but there is no evidence either way and + nothing in the template gives them meaning. Do not build on them. + +5. **The `thinking_budget` name collides across stacks and does not mean the same thing.** + vLLM calls it `thinking_token_budget`; llama.cpp calls it `reasoning_budget_tokens` with a + `thinking_budget_tokens` alias; SGLang calls it `thinking_budget` and reportedly does not + enforce it. A client that sends "the" thinking budget will silently get nothing on two of the + three. If we implement B2, accept every spelling. + +6. **Qwen's own agentic sizing advice assumes a feature most stacks lack.** "Set reasoning to + 262,144 and the final response to 131,072" presupposes *separate* caps for reasoning and + answer. Neither our server nor llama.cpp's has two caps; there is one `max_tokens`. This is + the failure mode SGLang issue #25536 describes — reasoning eats the whole budget and the + answer gets zero. **A budget mechanism (B2) is what makes a single `max_tokens` safe.** + +7. **The llama.cpp `preserve_reasoning` / `preserve_thinking` name mismatch** (A6) is stated + from reading the code, not from running it. `common/arg.cpp:3742` writes `preserve_reasoning`; + the Qwen 3.8 template reads `preserve_thinking` at line 116. I found no llama.cpp code that + translates between the two. Worth a five-minute empirical check before relying on it, and + worth an upstream issue if confirmed. + +8. **The quantization/trace-length papers are recent and I read abstracts, not full method + sections.** The 12-23% and 27-51% figures are the papers' own headline claims on their own + benchmarks, not measured on Qwen 3.8 or on our quant. The direction of the effect is + well-corroborated by two independent papers; the magnitudes are not ours until we measure. + +9. **Not checked:** whether Qwen 3.8 was RL-trained on the effort sentences specifically (the + card says "official support", which implies it, but no technical report was located), and + whether the `qwen4exp` Flash-Next weights behave identically to the dense 27B at the same + effort level despite the identical template. + +--- + +## Sources + +- — model card +- — the template +- +- +- — Flash-Next / `qwen4exp` model card +- — "This model cannot stop thinking" +- — "A crazy thinking model" +- — vLLM recipe +- — vLLM reasoning outputs +- — SGLang thinking_budget not enforced +- — Qwen3 card, for the `/think` `/no_think` history +- — Quantized Reasoning Models Think They Need to Think Longer +- — Quantization Inflates Reasoning +- — NoWait +- and — Antislop rollback sampler +- `D:\Work\llama.cpp` @ `6c84c7d5d` — `common/arg.cpp`, `common/common.h`, + `common/reasoning-budget.{h,cpp}`, `common/sampling.cpp`, `common/chat.cpp`, + `common/jinja/caps.cpp`, `tools/server/server-common.cpp`, `tools/server/server-schema.cpp`, + `tools/server/server-context.cpp`, `tools/server/README.md` +- `D:\Work\daScript` — `modules/dasLLAMA/THINKING.md`, + `modules/dasLLAMA/dasllama/dasllama_common.das`, `dasllama_chat.das`, + `dasllama_sampling.das`, `dasllama_arch_qwen35.das`, + `modules/dasLLAMA/performance/model_specs.das`, `utils/dasllama-server/openai_server.das` From 4ccd7ea08f99acedf6a44bad868feaf8c792f29a Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:22:18 -0700 Subject: [PATCH 018/123] HOW_TO_GET_SIDECAR.md: the mint, the provenance read, the export and the compare, as run on the zen4 box Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_GET_SIDECAR.md | 80 ++++++++++++++++++++------ 1 file changed, 62 insertions(+), 18 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md index 54ca6c06d7..8171d41bc9 100644 --- a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -112,23 +112,65 @@ into `plans/kernel_parity_pass.md`'s fact base (and the records store once the k ## 6. The mint -(The steps below are being run on the zen4 box as this document is written; they are filled in as -they prove out.) - -- The vehicle: a public 1B GGUF pulled straight to the box, no copying from the dev box - - `curl -L -o Llama-3.2-1B-Instruct-Q4_K_M.gguf https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf` - (808 MB). -- The mint: `bin/daslang -jit modules/dasLLAMA/benchmarks/lcpp_bench.das -- -m --tune` - the - full race of every family on this box (about 316 s on a c7a.4xlarge), writing - `modules/dasLLAMA/benchmarks/lcpp_bench.tune.json`. Read the noise verdict it prints. -- The export: `bin/daslang -jit modules/dasLLAMA/harness/export_tune_profile.das -- --sidecar - modules/dasLLAMA/benchmarks/lcpp_bench.tune.json` writes - `modules/dasLLAMA/performance/defaults/.tune-defaults.json` for this box's - `tune_cpu_class()`; `--class ` overrides the name. -- A class that does not exist yet (Intel's `x86-amx`): the chain lives in - `modules/dasLLVM/daslib/llvm_tune.das` (`tune_cpu_class`, `tune_class_chain`) and every feature a - `requires=` names sits in `TUNE_KNOWN_FEATURES` there; the new class goes above the class it - supersedes so a host resolves to the highest class it satisfies that has a shipped profile. +The vehicle is a public 1B GGUF pulled straight to the box - nothing copies from the dev box: + +``` +mkdir -p ~/models && cd ~/models +curl -L -o Llama-3.2-1B-Instruct-Q4_K_M.gguf https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf +cd ~/daScript +DAS_JOBQUE_THREADS=16 bin/daslang -jit modules/dasLLAMA/benchmarks/lcpp_bench.das -- -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --tune > ~/mint.log 2>&1 +``` + +808 MB, about a minute to fetch. `--tune` is the full race: every generator family +(`@tune begin name=q8_tile_gen ... @tune end ... winner= verdict=beats`), then the +`[tuned]` loop-hint kernels (`axpy`, `dot`, `rope_*`, `quantize_*`...), then +`confirm_e2e_prefill`; on the c7a.4xlarge the whole walk took 5-6 minutes. It writes +`modules/dasLLAMA/benchmarks/lcpp_bench.tune.json` beside the app and then re-launches the app to +apply it. The bench rows the re-launch would print are refused without `--for-debug-purposes` (a `-jit` +script run is not record-grade) - that refusal is expected here; the mint is done. + +Read the sidecar's provenance before anything else: + +``` +python3 -c 'import json;d=json.load(open("modules/dasLLAMA/benchmarks/lcpp_bench.tune.json"));p=d["provenance"];print(p["noise"],p["validation"],p["features"],len(d["kernels"]))' +``` + +`ok ok 49` is the pass: `noise` is the tuner's own drift verdict (a busy or thermally +unstable box says otherwise - re-mint, never edit), `validation` is every winner checked against its +fallback, `features` is the box's fingerprint of `TUNE_KNOWN_FEATURES` (zen4: +`avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni`), 49 the kernel count the scope demands. + +The export: + +``` +bin/daslang -jit modules/dasLLAMA/harness/export_tune_profile.das -- --sidecar modules/dasLLAMA/benchmarks/lcpp_bench.tune.json +git status --short modules/dasLLAMA/performance/defaults/ +``` + +It writes `performance/defaults/.tune-defaults.json` for this box's `tune_cpu_class()` +(`--class ` overrides, `--out ` relocates) with the box, engine sha and timing rows +stripped - a profile is kernel winners plus the provenance the adopt path checks. + +Compare before committing. A class that already ships a profile will differ in a few winners every +time - two mints of one class on two boxes disagree on the tie-class seats (the zen4 re-mint of +2026-09-01 flipped 4 of 49: `axpy` vec8_u2 -> vec8, `axpy_f16` -> plain, `rope_scaled_neox_tab` -> +plain, `q51q8_tile_gen` 512 -> 256-bit; the format tiles agreed). Commit a re-mint only when a seat +that carries a kernel family changed or new families exist; a tie flip is not a reason. + +``` +python3 - <<'EOP' +import json +n=json.load(open("modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json")) +import subprocess; o=json.loads(subprocess.check_output(["git","show","HEAD:modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json"])) +for k in sorted(set(n["kernels"])|set(o["kernels"])): + if n["kernels"].get(k)!=o["kernels"].get(k): print(k, o["kernels"].get(k), "->", n["kernels"].get(k)) +EOP +``` + +A class with no shipped profile yet (Intel's `x86-amx`) needs the class first: `tune_cpu_class()` and +the ladder in `tune_class_chain()` (`modules/dasLLVM/daslib/llvm_tune.das`), the new class above the +one it supersedes, and every feature a `requires=` names in `TUNE_KNOWN_FEATURES` there. Then the same +mint and export produce `.tune-defaults.json`, and that one IS committed. ## 7. What the commit must satisfy @@ -147,4 +189,6 @@ aws ec2 terminate-instances --instance-ids ## Boxes this walk ran on -- 2026-09-01 `c7a.4xlarge` (EPYC 9R14 zen4), `i-043725feb25086523`: sections 1-5 as written. +- 2026-09-01 `c7a.4xlarge` (EPYC 9R14 zen4, class `x86-vnni512`), `i-043725feb25086523`: sections 1-6 + as written; the TEST gate 65/65 ok; the re-mint agreed with the shipped profile on every format tile + and flipped four tie seats, so nothing was committed from it. From 1c0e29e24cf013dcb922ed27939c625b97613a53 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:22:39 -0700 Subject: [PATCH 019/123] the Qwen 3.8 memo through the Markdown ASCII gate Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- plans/qwen38_thinking_control.md | 114 +++++++++++++++---------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/plans/qwen38_thinking_control.md b/plans/qwen38_thinking_control.md index fbed5a9692..1073fee3df 100644 --- a/plans/qwen38_thinking_control.md +++ b/plans/qwen38_thinking_control.md @@ -32,7 +32,7 @@ everything here applies to both without change. ## (a) The mechanisms that actually exist -### A1. `enable_thinking` — a trained-in on/off switch, not an amount +### A1. `enable_thinking` - a trained-in on/off switch, not an amount Template spelling (`Qwen/Qwen3.8-27B/chat_template.jinja`, lines 163-170): @@ -51,7 +51,7 @@ Two things to notice. 1. With thinking **on** (the default), the template itself writes the opening `\n` into the generation prompt. The model never emits ``; it starts already inside the thought - block and only has to emit `` to leave it. This matters for any budget mechanism — + block and only has to emit `` to leave it. This matters for any budget mechanism - see A6 and B3. 2. With thinking **off**, the template writes an already-closed empty block `\n\n\n\n`. This is the same "prefill an empty closed block" trick Qwen3 used. @@ -68,7 +68,7 @@ wrapped in `chat_template_kwargs`. Source: and the raw template at . -### A2. `reasoning_effort` — the amount knob. It is a system-prompt sentence. +### A2. `reasoning_effort` - the amount knob. It is a system-prompt sentence. This is the only first-party control over *how much* the model thinks. Template lines 45-56, verbatim: @@ -92,17 +92,17 @@ Five facts follow directly from that code, and they are the most useful things i 1. **The whole mechanism is one sentence of English prepended to the system turn.** There is no special token, no logits change, no counter. The `reasoning_instructions` string is spliced - in ahead of the user's system content (template lines 58-86) — and if the request carries no + in ahead of the user's system content (template lines 58-86) - and if the request carries no system message at all, the template *creates* a system turn holding only that sentence (lines 81-85). 2. **`medium` injects nothing.** Only `xhigh` and `low` set the string; `medium` leaves it empty. So a request with `reasoning_effort="medium"` renders to a prompt that is byte-identical to a request with thinking on and no effort field. `medium` is the neutral baseline; `xhigh` and `low` are the two deviations from it. -3. **The default is the expensive end.** `reasoning_effort|default('xhigh')` — omitting the +3. **The default is the expensive end.** `reasoning_effort|default('xhigh')` - omitting the field gets you the "think carefully, validate assumptions, consider alternatives" sentence. -4. **Only three values are legal.** `low`, `medium`, `xhigh`. Anything else — including - OpenAI's own standard `"high"` — makes the template raise, and the request fails. This is a +4. **Only three values are legal.** `low`, `medium`, `xhigh`. Anything else - including + OpenAI's own standard `"high"` - makes the template raise, and the request fails. This is a sharp trap for an OpenAI-compatible front end, because `high` is the value most clients send. 5. Because it is only a prompt sentence, it is **advisory**. The model was post-trained to obey it, but nothing enforces it, and there is no upper bound on the trace length at any level. @@ -139,7 +139,7 @@ completion = client.chat.completions.create( vLLM's own recipe page for this model agrees on all of the above: . -### A3. `preserve_thinking` — history retention, which is an amount knob at second hand +### A3. `preserve_thinking` - history retention, which is an amount knob at second hand Template lines 111-119: @@ -157,7 +157,7 @@ Template lines 111-119: ``` Defaults to true. Note the retention is driven by a `reasoning_content` field on each historical -assistant message — so a client that strips reasoning before storing history silently gets +assistant message - so a client that strips reasoning before storing history silently gets `preserve_thinking` behaviour with empty thoughts (an empty `\n\n` per turn), which is neither of the two documented modes. The card claims retention improves KV-cache reuse and agent consistency (`README.md` line 472). Turning it off shortens the prompt but the card @@ -179,7 +179,7 @@ Checked directly against the downloaded files, all negative: - **No `/think` or `/no_think` soft switch.** Those were Qwen3-generation features documented on the Qwen3 model cards (). `grep -n "no_think\|/think"` finds no handling in the 3.8 template and no mention in either 3.8 model card. Whether the 3.8 - weights still react to those strings as plain prompt text is untested — see (c). + weights still react to those strings as plain prompt text is untested - see (c). - **No special reasoning-length token.** Nothing in the vocab named for effort or budget. ### A5. Token-level facts you need for any sampler-side work @@ -201,7 +201,7 @@ they are not stripped by `skip_special_tokens`; they are still atomic added toke `generation_config.json`: `temperature 1.0`, `top_k 20`, `top_p 0.95`, `eos_token_id [248046, 248044]`. -Recommended sampling per the card (`README.md` lines 252-253) — worth quoting because it differs +Recommended sampling per the card (`README.md` lines 252-253) - worth quoting because it differs between modes: - Thinking: `temperature=1.0, top_p=0.95, top_k=20, min_p=0.0, presence_penalty=0.0, repetition_penalty=1.0` - Non-thinking: `temperature=0.7, top_p=0.80, top_k=20, min_p=0.0, presence_penalty=1.5, repetition_penalty=1.0` @@ -218,7 +218,7 @@ The Flash-Next card says the same thing in prose (`README.md` line 350): *"Qwen3 supports controlling thinking behavior via `enable_thinking`, `preserve_thinking`, and `reasoning_effort`."* Same three effort levels, same `xhigh` default (line 384). -The llama.cpp commit `6c84c7d5d` that added `qwen4exp` is purely architectural — hyper-connections, +The llama.cpp commit `6c84c7d5d` that added `qwen4exp` is purely architectural - hyper-connections, gated delta net, MoE, PLE n-gram embeddings, UINT64 GGUF arrays. It touches no chat, template, or reasoning code. `grep -rn "xhigh"` across the whole llama.cpp tree hits exactly one line, the help text of `--reasoning-effort` (`common/arg.cpp:3711`). There is no Qwen3.8 template fixture under @@ -227,7 +227,7 @@ text of `--reasoning-effort` (`common/arg.cpp:3711`). There is no Qwen3.8 templa ### A6. What serving stacks bolt on -#### llama.cpp — a real reasoning-budget sampler (the most complete implementation seen) +#### llama.cpp - a real reasoning-budget sampler (the most complete implementation seen) `D:\Work\llama.cpp`, HEAD `6c84c7d5d`. @@ -241,12 +241,12 @@ IDLE -> COUNTING -> WAITING_UTF8 -> FORCING -> DONE - `COUNTING`: decrement per token, watching for a natural end sequence. - `WAITING_UTF8`: budget spent, but let the current multi-byte character finish. - `FORCING`: emit the forced sequence token by token. `common/reasoning-budget.cpp:166-186` is - the whole enforcement — every logit except the one forced token is set to `-INFINITY`. + the whole enforcement - every logit except the one forced token is set to `-INFINITY`. - `DONE`: passthrough. It **re-arms** on a new start tag (`reasoning-budget.cpp:147-161`), because some models open several `` blocks in one reply. **The forced sequence** is `reasoning_budget_message` tokens followed by the first end tag -(`tools/server/server-schema.cpp:415-427`). That is exactly the "Considering the limited time…" +(`tools/server/server-schema.cpp:415-427`). That is exactly the "Considering the limited time..." pattern: you supply the sentence, the server prepends it to `` and forces the lot. **CLI flags** (`common/arg.cpp`): @@ -265,14 +265,14 @@ Environment aliases exist for each (`LLAMA_ARG_REASONING`, `LLAMA_ARG_REASONING_ **Per-request HTTP fields** (`tools/server/server-common.cpp:1313-1378`): -- `chat_template_kwargs: {"enable_thinking": bool, ...}` — merged over the server defaults. -- `reasoning_effort: ""` — the OpenAI field. `"none"` is special-cased to +- `chat_template_kwargs: {"enable_thinking": bool, ...}` - merged over the server defaults. +- `reasoning_effort: ""` - the OpenAI field. `"none"` is special-cased to `enable_thinking = false` (line 1326-1328); any other non-empty string is forwarded as the template kwarg (line 1330). - `reasoning_budget_tokens`, alias `thinking_budget_tokens` (line 1365-1366), falling back to the server default. - `reasoning_budget_message` (line 1375). -- `reasoning_control: bool` (line 1376) — arms the sampler so it can be forced later at runtime. +- `reasoning_control: bool` (line 1376) - arms the sampler so it can be forced later at runtime. **Runtime early-stop.** `POST /v1/chat/completions/control` with `{"id": , "action": "reasoning_end"}` forces the in-flight completion out of its @@ -283,7 +283,7 @@ This is a "stop thinking, answer now" button for a UI. **Two llama.cpp findings that bear on Qwen 3.8 specifically:** 1. **The budget arms correctly despite the template pre-filling ``.** Recall from A1 that - the model never emits `` — the template does. llama.cpp handles this: the generation + the model never emits `` - the template does. llama.cpp handles this: the generation prompt is re-tokenized into `prefill_tokens` (`common/sampling.cpp:279-292`) and those tokens are fed through `llama_sampler_accept` on the budget sampler at construction (`common/sampling.cpp:318-322`), so the `` in the prompt moves the state machine @@ -314,12 +314,12 @@ supported. #### vLLM -- Sampling parameter `thinking_token_budget` — per-request reasoning token limit. +- Sampling parameter `thinking_token_budget` - per-request reasoning token limit. - `--reasoning-parser qwen3` is required for Qwen 3.8, because the template opens every assistant turn with `` and without the parser the whole thought block lands in `content` (). - `--reasoning-config '{"reasoning_start_str": "", "reasoning_end_str": "I have to give the - solution based on the reasoning directly now."}'` — this is where the "limited time" + solution based on the reasoning directly now."}'` - this is where the "limited time" style closer comes from. The docs say putting transitional language in `reasoning_end_str` makes the termination "more natural". - `--default-chat-template-kwargs '{"enable_thinking": false}'` for a server-wide default; @@ -331,7 +331,7 @@ supported. Exposes `chat_template_kwargs` and `separate_reasoning: true` (which populates `reasoning_content`), plus a `thinking_budget`. The budget is reported **not enforced** on this -model generation: sgl-project/sglang issue #25536, "thinking_budget not enforced for Qwen3.6 — +model generation: sgl-project/sglang issue #25536, "thinking_budget not enforced for Qwen3.6 - reasoning consumes all max_tokens", where `thinking_budget: 200` still produced roughly 1400 reasoning tokens and left nothing for the answer (). Treat SGLang's budget as unreliable. @@ -350,14 +350,14 @@ This is directly relevant to us because we run this model quantized. - *"Quantized Reasoning Models Think They Need to Think Longer, but They Do Not"* (). Abstract, verbatim in the load-bearing part: *"aggressive - PTQ reduces accuracy while increasing chain-of-thought (CoT) length… in up to 52% of the + PTQ reduces accuracy while increasing chain-of-thought (CoT) length... in up to 52% of the quantized models' failures, models reach the right answer in intermediate reasoning steps but do - not output it as a final answer… Positions with high KL divergence correlate strongly with high + not output it as a final answer... Positions with high KL divergence correlate strongly with high next-token entropy, and at these positions quantized models disproportionately sample overthinking markers such as "wait", "but", and "alternatively". We show that simply introducing a training-free logit penalty on a curated set of overthinking markers can reduce CoT length by 12--23% while preserving or improving accuracy across 5 models (1.5B-32B parameters), 3 - quantization methods, and 5 benchmarks… Overthinking errors produced by quantized models are + quantization methods, and 5 benchmarks... Overthinking errors produced by quantized models are particularly reduced by up to 58%."* - *"Quantization Inflates Reasoning: Token Inflation as a Hidden Cost of Low-Bit Reasoning Models"* (). INT4/INT3 can preserve accuracy while inflating reasoning @@ -370,7 +370,7 @@ same `reasoning_effort`, and the extra length is disproportionately made of "wai is the paper's own remedy and it is cheap. **Banning reconsideration phrases outright.** *"Wait, We Don't Need to 'Wait'! Removing Thinking -Tokens Improves Reasoning Efficiency"* (NoWait), — suppresses +Tokens Improves Reasoning Efficiency"* (NoWait), - suppresses explicit self-reflection tokens (`Wait`, `Hmm`) at decode time and reports a 27%-51% shorter chain of thought across five R1-style model series and ten benchmarks with utility preserved. @@ -379,10 +379,10 @@ chain of thought across five R1-style model series and ten benchmarks with utili ). Unlike a logit ban it works on *multi-token phrases*: it keeps the inference trace, and when a banned pattern completes it backtracks to the token where the pattern began, downweights the offending continuation, and -resamples. The suppression is soft — `p_new = p_old * 10^(-10s)` with ban strength `s` in `[0,1]`, +resamples. The suppression is soft - `p_new = p_old * 10^(-10s)` with ban strength `s` in `[0,1]`, `s=1` being a hard ban. It scales to 8000+ patterns (direct token banning caps out near 2000) and accepts regexes. The cost is real: 69%-96% output slowdown in the bad cases. This is the only -mechanism in the set that can ban a phrase like `", but wait"` precisely — a plain logit ban on +mechanism in the set that can ban a phrase like `", but wait"` precisely - a plain logit ban on the `wait` token cannot distinguish `", but wait"` from a legitimate `wait`. For comparison, llama.cpp itself has **no** backtracking sampler (`grep -rni "antislop\|backtrack"` @@ -398,7 +398,7 @@ Our stack, for reference: chat rendering is `modules/dasLLAMA/dasllama/dasllama_ per-family `ChatTemplate` data declared in `dasllama_arch_*.das` / `dasllama_common.das:1120-1150`; sampling is `modules/dasLLAMA/dasllama/dasllama_sampling.das:23-32` (`SamplingParams`: temp, top_k, top_p, -min_p, repetition/presence/frequency penalties — no logit bias, no forced tokens, no phrase ban); +min_p, repetition/presence/frequency penalties - no logit bias, no forced tokens, no phrase ban); the OpenAI server is `utils/dasllama-server/openai_server.das`. The design overview is `modules/dasLLAMA/THINKING.md`. We already carry a Qwen3.8-27B GGUF spec (`modules/dasLLAMA/performance/model_specs.das:346-351`, `Qwen3.8-27B-UD-Q4_K_M.gguf`, @@ -422,13 +422,13 @@ Map the three rungs straight onto Qwen's three levels: Concrete steps: 1. **Add an effort field to `ChatTemplate`.** The cleanest shape given our existing data-driven - design is a three-slot string set — e.g. `effort_low : string`, `effort_high : string`, with + design is a three-slot string set - e.g. `effort_low : string`, `effort_high : string`, with empty meaning "inject nothing", which is literally how Qwen encodes `medium`. Render it into the system turn ahead of the user's system content, matching template lines 58-86, including the case where there is no user system prompt and we must synthesize a system turn holding only the sentence. 2. **Add a `reasoning_effort` field to the server.** Add `"reasoning_effort"` to `CHAT_FIELDS` - (`utils/dasllama-server/openai_server.das:1556-1559`) — today it is silently warned about as an + (`utils/dasllama-server/openai_server.das:1556-1559`) - today it is silently warned about as an unsupported field. Also accept it inside `chat_template_kwargs` (the loop at line 2521-2527 currently warns on every key but `enable_thinking`). 3. **Decide the out-of-range policy, and do not copy Qwen's.** Qwen's template hard-fails on @@ -437,7 +437,7 @@ Concrete steps: `server-common.cpp:1326`), and reject the rest with a 400 naming the legal set. This is a deliberate divergence from llama.cpp, which passes the string through and lets the template blow up. -4. **Keep it a per-family declaration, not a Qwen special case** — some families have no effort +4. **Keep it a per-family declaration, not a Qwen special case** - some families have no effort sentence at all and should render nothing regardless of the request. Caveat to state in the UI: this is a *request*, not a cap. It has no worst case. @@ -451,19 +451,19 @@ Concrete steps: 1. **Add to `SamplingParams`:** `reason_budget : int64` (`-1` off, `0` = end immediately, `N` = token cap), plus the forced-token list and the optional closer message. -2. **Count inside the block.** The generation loop knows whether it is inside the thought span — +2. **Count inside the block.** The generation loop knows whether it is inside the thought span - `dasllama_chat.das` already has the reply-side matcher (`ThinkStream` / `make_think_stream_`/`think_feed_`, `dasllama_chat.das:1054-1057`) for exactly this. Arm the counter when the block opens. **Critical**: with the Qwen 3.8 shape the block is opened by the *prompt*, not by a sampled token, so the counter has to be armed at prompt-render time when - `think_open` was prefilled — this is the same bug llama.cpp avoids by feeding the prefill + `think_open` was prefilled - this is the same bug llama.cpp avoids by feeding the prefill tokens through the sampler (`common/sampling.cpp:318-322`). 3. **Force on exhaustion.** Set every logit but the forced token to `-inf` for as many steps as the forced sequence is long. If a closer message is configured, force its tokens first and - `` (248069) last — this is what makes the transition read naturally rather than as an + `` (248069) last - this is what makes the transition read naturally rather than as an abrupt cut. Mirror llama.cpp's UTF-8 grace state (`reasoning-budget.h:14`) so we do not sever a multi-byte character. -4. **Re-arm on a second ``** (`reasoning-budget.cpp:147-161`) — the model can open more +4. **Re-arm on a second ``** (`reasoning-budget.cpp:147-161`) - the model can open more than one block per reply. 5. **Server field:** accept `reasoning_budget_tokens` with alias `thinking_budget_tokens` (llama.cpp's spelling, `server-common.cpp:1365-1366`) plus `reasoning_budget_message`. @@ -497,15 +497,15 @@ penalty gets most of the benefit at no throughput cost. - **`dasllama_arch_qwen38.das` does not exist.** Qwen 3.8 is ChatML-shaped so `chatml_chat` is the right base, but three things differ from our Qwen3/3.5 wiring: (i) the generation prompt - must prefill `\n` when thinking is on — our `chatml_chat` does not + must prefill `\n` when thinking is on - our `chatml_chat` does not (`dasllama_common.das:4642-4643`), and `ChatTemplate` already has the field for it, `assistant_open_think`, added for gemma-4 (`dasllama_common.das:1135`); (ii) the tool block is - Qwen3-Coder XML (``), **not** the Hermes JSON our + Qwen3-Coder XML (``), **not** the Hermes JSON our `dasllama_arch_qwen35.das:44` installs; (iii) the effort sentence from B1. - **History retention conflicts with Qwen's default.** `THINKING.md` records that our `respond_` stores history **reasoning-stripped**. Qwen 3.8 defaults to `preserve_thinking = true` and renders `\n{reasoning_content}\n` for every historical assistant turn. Stripping - leaves us rendering empty thought blocks — neither documented mode. We should either carry + leaves us rendering empty thought blocks - neither documented mode. We should either carry `reasoning_content` in history and honour `preserve_thinking`, or render the non-preserving branch (template line 119) deliberately. Right now we do neither on purpose. @@ -530,8 +530,8 @@ penalty gets most of the benefit at no throughput cost. 3. **`medium` being prompt-identical to "no instruction" is my reading of the template, not a documented claim.** It follows directly from lines 45-56 (the string is only set for `xhigh` - and `low`), but Qwen never says it. It implies the model's *trained* default behaviour — what - it does with no effort sentence — is the `medium` behaviour, and that `xhigh` being the + and `low`), but Qwen never says it. It implies the model's *trained* default behaviour - what + it does with no effort sentence - is the `medium` behaviour, and that `xhigh` being the template default means the shipped default is deliberately above the trained baseline. That inference is untested. @@ -549,7 +549,7 @@ penalty gets most of the benefit at no throughput cost. 6. **Qwen's own agentic sizing advice assumes a feature most stacks lack.** "Set reasoning to 262,144 and the final response to 131,072" presupposes *separate* caps for reasoning and answer. Neither our server nor llama.cpp's has two caps; there is one `max_tokens`. This is - the failure mode SGLang issue #25536 describes — reasoning eats the whole budget and the + the failure mode SGLang issue #25536 describes - reasoning eats the whole budget and the answer gets zero. **A budget mechanism (B2) is what makes a single `max_tokens` safe.** 7. **The llama.cpp `preserve_reasoning` / `preserve_thinking` name mismatch** (A6) is stated @@ -572,26 +572,26 @@ penalty gets most of the benefit at no throughput cost. ## Sources -- — model card -- — the template +- - model card +- - the template - - -- — Flash-Next / `qwen4exp` model card -- — "This model cannot stop thinking" -- — "A crazy thinking model" -- — vLLM recipe -- — vLLM reasoning outputs -- — SGLang thinking_budget not enforced -- — Qwen3 card, for the `/think` `/no_think` history -- — Quantized Reasoning Models Think They Need to Think Longer -- — Quantization Inflates Reasoning -- — NoWait -- and — Antislop rollback sampler -- `D:\Work\llama.cpp` @ `6c84c7d5d` — `common/arg.cpp`, `common/common.h`, +- - Flash-Next / `qwen4exp` model card +- - "This model cannot stop thinking" +- - "A crazy thinking model" +- - vLLM recipe +- - vLLM reasoning outputs +- - SGLang thinking_budget not enforced +- - Qwen3 card, for the `/think` `/no_think` history +- - Quantized Reasoning Models Think They Need to Think Longer +- - Quantization Inflates Reasoning +- - NoWait +- and - Antislop rollback sampler +- `D:\Work\llama.cpp` @ `6c84c7d5d` - `common/arg.cpp`, `common/common.h`, `common/reasoning-budget.{h,cpp}`, `common/sampling.cpp`, `common/chat.cpp`, `common/jinja/caps.cpp`, `tools/server/server-common.cpp`, `tools/server/server-schema.cpp`, `tools/server/server-context.cpp`, `tools/server/README.md` -- `D:\Work\daScript` — `modules/dasLLAMA/THINKING.md`, +- `D:\Work\daScript` - `modules/dasLLAMA/THINKING.md`, `modules/dasLLAMA/dasllama/dasllama_common.das`, `dasllama_chat.das`, `dasllama_sampling.das`, `dasllama_arch_qwen35.das`, `modules/dasLLAMA/performance/model_specs.das`, `utils/dasllama-server/openai_server.das` From 17c44b2bddf1e96f23d7d58217d458aa1898b10c Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:23:17 -0700 Subject: [PATCH 020/123] kernel parity: the zen4 model-level stamp and the repack caveat on the ladder's k-quant rows; the how-to's box log Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_GET_SIDECAR.md | 5 ++++- plans/kernel_parity_pass.md | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md index 8171d41bc9..210e53c29a 100644 --- a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -191,4 +191,7 @@ aws ec2 terminate-instances --instance-ids - 2026-09-01 `c7a.4xlarge` (EPYC 9R14 zen4, class `x86-vnni512`), `i-043725feb25086523`: sections 1-6 as written; the TEST gate 65/65 ok; the re-mint agreed with the shipped profile on every format tile - and flipped four tie seats, so nothing was committed from it. + and flipped four tie seats, so nothing was committed from it. Model-level sanity on the vehicle + (`lcpp_bench -m -p 512 -n 128 -r 3 --for-debug-purposes --ref ~/llama.cpp/build-clean-cpu/bin/llama-bench + --ref-flavor clean-cpu --ref-no-affinity`, 16 threads): pp512 1172 vs 927, tg128 88.7 vs 86.0. + Wall clock from launch to terminate: about 90 minutes, of which the build is 12 and the mint 6. diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 481e5db34d..5da80e142e 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -87,6 +87,11 @@ iq3xxs 0.88, iq3s 0.81, iq2xxs 0.81, iq2xs 0.79, iq2s 0.77; every tile 2.1x-12x the work, the 512-bit VNNI seat pays (k4 1.22x on zen2 -> 2.25x); the five grid formats lose ground because the reference's grid kernels gain ~1.5x from AVX-512 while ours are bound by the scalar per-dword gather (iq2s 4898 us on zen4 vs 5059 on zen2) - the ARM memo's diagnosis, on x86. +Model level on the zen4 (Q4_K_M 1B, 16 threads, debug-jit rows): pp512 1172 vs 927 (1.26x), tg128 88.7 +vs 86.0 (1.03x). The kernel ladder's k4 2.25x is against the reference's fallback vec_dot; a model run +uses its AVX2/AVX-512 repack 8x8 GEMV for q4_K - so the ladder's k4/k5/k6/q4_0/q8_0/iq4_nl/q2_K rows +overstate against real serving where the reference repacks (repack.cpp covers those eight types); the +grid formats and q3_K have no repack path and their rows are the fair ones. zen2 reading: every tile row is ahead (1.17x-7.07x). Decode tails were k3 0.80x, k6 0.90x, k5 0.91x; iq3xxs, k2, iq2s, iq2xxs at 0.99-1.00 (inside the noise band). The 4-bit class (q40, q51, iq4xs, iq4nl, k4, q8, mx4) From f6e6b128d81d5660058fdc2788ade4799eb3cdd8 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 03:39:36 -0700 Subject: [PATCH 021/123] kq_kernel_bench: 64-byte-aligned planes (Intel splits a misaligned 64-byte load - k6 decode read 2x slow on Granite Rapids), and the q8 arm arms the AMX witness before racing Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../benchmarks/matmul/kq_kernel_bench.das | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index a477d6542e..da2709374b 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -165,11 +165,16 @@ struct Planes { y : array } +let ALIGN = 64l // the image's planes are page-aligned (ARCHITECTURE_IMAGE.md, image-page-alignment); a 64-byte load split across cache lines is what Intel penalizes 2x + def sized(var a : array; n : int64) { - a |> reserve(n) - a |> resize(n) + a |> reserve(n + ALIGN) + a |> resize(n + ALIGN) } +//! the first 64-byte-aligned element of an over-allocated plane +def aligned_start(p : void?) : int64 => (ALIGN - int64(intptr(p) & uint64(ALIGN - 1l))) & (ALIGN - 1l) + def fill_planes(var p : Planes; sh : PlaneShape; n, d, ntok : int64) { p.kq |> sized(d * sh.wq_row) p.tkq |> sized(d * sh.tq_row) @@ -267,7 +272,7 @@ def time_rows(var stats : array; rounds : int; blk : block<(row : int) def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; var p : Planes; nsb : int64) { let n = int64(cfg.n) let d = int64(cfg.d) - let plane_bytes = long_length(p.kq) + long_length(p.ks) + let plane_bytes = long_length(p.kq) + long_length(p.ks) - 2l * ALIGN var vs <- gemv_variants(f) var stats : array var fns : array @@ -278,12 +283,12 @@ def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va } } unsafe { - let kqp = addr(p.kq[0]) - let ksp = addr(p.ks[0]) - let xqp = addr(p.xq[0]) - let xsp = addr(p.xs[0]) - let xbsp = addr(p.xbs[0]) - var yp = addr(p.y[0]) + let kqp = addr(p.kq[aligned_start(addr(p.kq[0]))]) + let ksp = addr(p.ks[aligned_start(addr(p.ks[0]))]) + let xqp = addr(p.xq[aligned_start(addr(p.xq[0]))]) + let xsp = addr(p.xs[aligned_start(addr(p.xs[0])) / 4l]) + let xbsp = addr(p.xbs[aligned_start(addr(p.xbs[0])) / 4l]) + var yp = addr(p.y[aligned_start(addr(p.y[0])) / 4l]) time_rows(stats, cfg.rounds) $(row : int) { invoke(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) } @@ -302,7 +307,7 @@ def bench_tile_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va let ntok = int64(cfg.ntok) let qrow = sh.tq_row let srow = sh.ws_row - let plane_bytes = long_length(p.tkq) + long_length(p.ks) + let plane_bytes = long_length(p.tkq) + long_length(p.ks) - 2l * ALIGN var mrs <- companion_ints(layout_variants(f)) var vs <- tile_variants(f) var stats : array @@ -322,12 +327,12 @@ def bench_tile_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va mrOf |> push(mr) } unsafe { - let kqp = addr(p.tkq[0]) - let ksp = addr(p.ks[0]) - let xqp = addr(p.xq[0]) - let xsp = addr(p.xs[0]) - let xbsp = addr(p.xbs[0]) - var yp = addr(p.y[0]) + let kqp = addr(p.tkq[aligned_start(addr(p.tkq[0]))]) + let ksp = addr(p.ks[aligned_start(addr(p.ks[0]))]) + let xqp = addr(p.xq[aligned_start(addr(p.xq[0]))]) + let xsp = addr(p.xs[aligned_start(addr(p.xs[0])) / 4l]) + let xbsp = addr(p.xbs[aligned_start(addr(p.xbs[0])) / 4l]) + var yp = addr(p.y[aligned_start(addr(p.y[0])) / 4l]) time_rows(stats, cfg.rounds) $(row : int) { let mr = mrOf[row] for (g in range64(d / mr)) { @@ -411,14 +416,15 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes let d = int64(cfg.d) let ntok = int64(cfg.ntok) let name = mx4 ? MX4_NAME : "q8" - let plane_bytes = long_length(p.kq) + long_length(p.ks) + let plane_bytes = long_length(p.kq) + long_length(p.ks) - 2l * ALIGN + q8q8_family_live() // the family's witness: on an AMX box it takes the per-process tile permission the amx seats need (SIGILL without it) unsafe { - let kqp = addr(p.kq[0]) - let ksp = addr(p.ks[0]) - let xqp = addr(p.xq[0]) - let xsp = addr(p.xs[0]) - let xbsp = addr(p.xbs[0]) - var yp = addr(p.y[0]) + let kqp = addr(p.kq[aligned_start(addr(p.kq[0]))]) + let ksp = addr(p.ks[aligned_start(addr(p.ks[0]))]) + let xqp = addr(p.xq[aligned_start(addr(p.xq[0]))]) + let xsp = addr(p.xs[aligned_start(addr(p.xs[0])) / 4l]) + let xbsp = addr(p.xbs[aligned_start(addr(p.xbs[0])) / 4l]) + var yp = addr(p.y[aligned_start(addr(p.y[0])) / 4l]) if (!cfg.no_gemv) { var stats : array var fnsQ : array From 6c9c07fe1534c8b65fe84f0d193eddc4b1851849 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 04:05:26 -0700 Subject: [PATCH 022/123] the x86-amx defaults profile, minted on a Granite Rapids c8i.4xlarge; kq_kernel_bench: normal scale bytes (Intel's denormal penalty bent the q8/mx4 tile rows) and the AMX witness variants armed before racing The class profile differs from x86-vnni512 in 15 of 49 winners, all [tuned] loop-hint kernels preferring vec16 on the Xeon 6975P-C; every generator tile keeps the 512-bit VNNI seat - the AMX tiles raced and lost the q8q8 family. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../benchmarks/matmul/kq_kernel_bench.das | 10 ++- .../defaults/x86-amx.tune-defaults.json | 69 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index da2709374b..feed9612ca 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -56,6 +56,7 @@ typedef LayoutFn = function<() : int> let TILE_TOKENS = 4l // tokens one kq / q51 / mx4 tile call covers (q8's tokstep companion says its own) let MX4_NAME = "mx4" // mxfp4 has no KqFmt row; it rides the q8q8 grid as a companion +let SCALE_BYTE = uint8(0x3C) // as f16 0x3C3C = 1.06, as f32 0x3C3C3C3C = 0.0115, as e8m0 2^-67, as a 6-bit sub-scale 60 - normal in every scale form def fmt_of_name(name : string) : KqFmt { for (f in type) { @@ -193,8 +194,7 @@ def fill_planes(var p : Planes; sh : PlaneShape; n, d, ntok : int64) { b = uint8(seed >> 24u) } for (b in p.ks) { - seed = seed * 1664525u + 1013904223u - b = uint8(seed >> 24u) + b = SCALE_BYTE // scale planes hold normal numbers in every scale form, never random bytes: Intel runs denormal math ~100x slower, AMD does not, and a random byte pattern is a denormal or an infinity often enough to bend a table } for (q in p.xq) { seed = seed * 1664525u + 1013904223u @@ -417,7 +417,11 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes let ntok = int64(cfg.ntok) let name = mx4 ? MX4_NAME : "q8" let plane_bytes = long_length(p.kq) + long_length(p.ks) - 2l * ALIGN - q8q8_family_live() // the family's witness: on an AMX box it takes the per-process tile permission the amx seats need (SIGILL without it) + var wvs <- q8q8_family_live_variants() // the family's witness rows: the amx one takes the per-process tile permission its tiles need (SIGILL without it), as the probe does + for (w in wvs) { + invoke(w._1) + } + delete wvs unsafe { let kqp = addr(p.kq[aligned_start(addr(p.kq[0]))]) let ksp = addr(p.ks[aligned_start(addr(p.ks[0]))]) diff --git a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json new file mode 100644 index 0000000000..831a423d80 --- /dev/null +++ b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json @@ -0,0 +1,69 @@ +{ + "kernels" : { + "add_inplace" : "vec16", + "cvt_f32_to_f16" : "vec16", + "rope_scaled_neox_tab" : "vec16", + "q51q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "softmax" : "vec8_u2", + "mul_inplace" : "vec16", + "quantize_q8_0_bs_into_ptr" : "vec16_u2", + "iq2xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8kv" : "vec8_u2", + "dot_q8q8" : "vec16_u2", + "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "quantize_q8kv_row" : "plain", + "axpy_f16" : "vec16", + "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "axpy_tq4kv" : "vec8_u2", + "cvt_tq4kv_to_f32" : "vec8_u2", + "axpy" : "vec16", + "dot_q8q8kv" : "vec16_u2", + "dot_mx4q8" : "u2", + "softmax_sink" : "vec8_u2", + "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8q8_laneq4x4" : "", + "dot_bf16" : "vec8_u2", + "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "add_scale_inplace" : "vec8_u2", + "cvt_q8kv_to_f32" : "vec16_u2", + "axpy_q8kv" : "vec16_u2", + "dot_q8q8_f16s" : "vec16", + "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", + "quantize_q8_0_into_ptr" : "plain", + "k4q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k3q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k2q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "gemm_f32_uk_4x16" : "u2", + "dot_q51e" : "vec16", + "dot_f16" : "vec16", + "cvt_f16_to_f32" : "vec16", + "dot" : "vec8_u2", + "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8tq4kv" : "vec16", + "scale_inplace" : "vec16", + "dot_q4" : "vec8_u2", + "quantize_tq4kv_row" : "plain", + "copy_floats" : "vec8_u2", + "rmsnorm" : "vec16" + }, + "provenance" : { + "validation" : "ok", + "noise_probes" : "start cv 0.20%; mid1 cv 0.06%; mid2 cv 0.09%; end cv 0.19%", + "platform" : "linux", + "noise_floor_cv_pct" : "0.20", + "features" : "avx2;f16c;fma;sse4.2;avxvnni;avx512f;avx512bw;avx512vl;avx512vnni;amx-tile;amx-int8", + "class" : "x86-amx", + "written" : "2026-09-01T11:02:47.521Z", + "validation_demoted" : "0", + "mode" : "normal", + "dasllama_version" : "15", + "origin" : "profile", + "noise" : "ok", + "arch" : "x86_64", + "validation_max_drift_pct" : "1.97" + } +} \ No newline at end of file From 612d7accf34715e53d155ce49f98c319433218f9 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 04:06:10 -0700 Subject: [PATCH 023/123] HOW_TO_GET_SIDECAR.md: the new-class walk as run on the Intel box, and the three box lessons (alignment, denormals, the AMX grant) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_GET_SIDECAR.md | 28 ++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md index 210e53c29a..1ec3378b52 100644 --- a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -167,10 +167,27 @@ for k in sorted(set(n["kernels"])|set(o["kernels"])): EOP ``` -A class with no shipped profile yet (Intel's `x86-amx`) needs the class first: `tune_cpu_class()` and -the ladder in `tune_class_chain()` (`modules/dasLLVM/daslib/llvm_tune.das`), the new class above the -one it supersedes, and every feature a `requires=` names in `TUNE_KNOWN_FEATURES` there. Then the same -mint and export produce `.tune-defaults.json`, and that one IS committed. +A class with no shipped profile yet (Intel's `x86-amx`, born 2026-09-01) needs the class first, on the +branch the box clones: `tune_cpu_class()` and the ladder in `tune_class_chain()` +(`modules/dasLLVM/daslib/llvm_tune.das`), the new class above the one it supersedes, and every feature +a `requires=` names in `TUNE_KNOWN_FEATURES` there (`amx-int8`, `amx-tile` were already listed). Until +its profile ships, a box of the new class adopts the class below it (the chain) and races only the +seats that class could not answer. Then the same mint and export produce `.tune-defaults.json`, +and that one IS committed - the Intel one differed from `x86-vnni512` in 15 of 49 winners, all +`[tuned]` loop-hint kernels preferring `vec16`, while every generator tile kept the 512-bit VNNI seat +(the AMX tiles raced and lost the q8q8 family). + +Three things the boxes taught that the walk now carries: + +- **Alignment.** Intel splits a 64-byte vector load that crosses a cache line and pays for it; AMD + barely does. The engine's image planes are page-aligned, but a bench that hands kernels 16-byte-aligned + arrays reads 2x slow on Intel and true on zen4 - `kq_kernel_bench` aligns its planes to 64 bytes. +- **Denormals.** Random bytes in a scale plane are denormals or infinities often enough to bend a + table on Intel (~100 cycles a denormal op) and not on AMD: the q8/mx4 tile rows read 6x slow until the + bench filled scale planes with a byte that is a normal number in every scale form. +- **AMX permission.** A tile instruction before the per-process `arch_prctl` grant is SIGILL. The + family's witness performs the grant; in tune mode call the witness *variants* (the AMX row does it), + as `gen_tune_probe` does - the unstamped base does not. ## 7. What the commit must satisfy @@ -195,3 +212,6 @@ aws ec2 terminate-instances --instance-ids (`lcpp_bench -m -p 512 -n 128 -r 3 --for-debug-purposes --ref ~/llama.cpp/build-clean-cpu/bin/llama-bench --ref-flavor clean-cpu --ref-no-affinity`, 16 threads): pp512 1172 vs 927, tg128 88.7 vs 86.0. Wall clock from launch to terminate: about 90 minutes, of which the build is 12 and the mint 6. +- 2026-09-01 `c8i.4xlarge` (Xeon 6975P-C Granite Rapids, class `x86-amx`), `i-0fb77cb72129e36a2`: + sections 1-7; the TEST gate 65/65 ok with the AMX leg; the class was added on the branch before + launch; the minted `x86-amx.tune-defaults.json` is the shipped one. From d99bce97d7ac31ceef06e8d1bdc6f9f402252b0b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 04:15:16 -0700 Subject: [PATCH 024/123] kernel parity: the Intel ladder (Granite Rapids, x86-amx) with its two bench artifacts fixed and the open items Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- plans/kernel_parity_pass.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 5da80e142e..40e5cb1827 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -87,6 +87,18 @@ iq3xxs 0.88, iq3s 0.81, iq2xxs 0.81, iq2xs 0.79, iq2s 0.77; every tile 2.1x-12x the work, the 512-bit VNNI seat pays (k4 1.22x on zen2 -> 2.25x); the five grid formats lose ground because the reference's grid kernels gain ~1.5x from AVX-512 while ours are bound by the scalar per-dword gather (iq2s 4898 us on zen4 vs 5059 on zen2) - the ARM memo's diagnosis, on x86. +The Intel ladder (AWS c8i.4xlarge, Xeon 6975P-C Granite Rapids, one thread, the new x86-amx class - +its profile minted here and shipped; TEST 65/65 ok with the AMX leg; the bench with 64-byte-aligned +planes and normal scale bytes): decode q51 5.07, mx4 3.26, q8 3.21, iq4xs 2.99, q40 2.92, iq4nl 2.88, +k3 2.47, k2 2.20, k5 1.94, k4 1.54, iq2xs 1.15, iq3s 1.10, iq2xxs 1.10, iq3xxs 0.94, iq2s 0.91, +k6 0.39 (bimodal on this box: 2.1 ms in two runs, 5.8-6.7 in two - allocation-dependent past 64-byte +alignment, open); every tile 2.06x-14.2x ahead. The reference exe's own rows swung up to 2x between runs +on the VM (q8_0 4036 -> 7487), so single-run Intel ratios carry that error bar. Two bench artifacts were +found and fixed on this box: 16-byte-aligned planes (Intel splits a 64-byte load across a cache line - +k6 6703 -> 2234 us) and random scale bytes (denormal math - q8 tile 1022 -> 155 ms). Open: the bench's +tune-mode q8 tile SIGILLs on the AMX box even after invoking the witness variants; the tuner's own arming +works (the mint raced the AMX tiles), so the bench arms AMX differently from gen_tune_probe - find how. + Model level on the zen4 (Q4_K_M 1B, 16 threads, debug-jit rows): pp512 1172 vs 927 (1.26x), tg128 88.7 vs 86.0 (1.03x). The kernel ladder's k4 2.25x is against the reference's fallback vec_dot; a model run uses its AVX2/AVX-512 repack 8x8 GEMV for q4_K - so the ladder's k4/k5/k6/q4_0/q8_0/iq4_nl/q2_K rows @@ -191,6 +203,9 @@ masked negate already beats llama.cpp's own mask expansion. Fallback if deferred bytes (size-neutral for iq2xxs/iq3xxs, +6% for iq2xs; touches the plane, both ISAs) -> 0.72-0.98x only. Also from the memo: the column-read-vs-byte-read decision was made on x86 and should be a perm seat. +Bench on AMX boxes: the tune-mode q8 tile SIGILL (above) - read how gen_tune_probe reaches the grant +(q8q8_family_live_variants + the amx cfg companion?) and do the same; k6 bimodality on Granite Rapids. + Vulkan pp (gap 1): 1. Budget split (measurement, no kernel edit) - is the GEMM the 30%? 2. `shAscales`-style scale hoist; 3. scale plane interleave; 4. N padding; 5+. the rest of the memo's list. From cd2f427637e9c1c240ba47cef7e8ade69cbcde3c Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 04:16:26 -0700 Subject: [PATCH 025/123] kernel parity: the zen2 ladder re-run with the fixed bench Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- plans/kernel_parity_pass.md | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 40e5cb1827..54aef00f4e 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -50,8 +50,33 @@ proven fact per push; research before any kernel edit (the two memos below). | iq2xs | 11490 (50.1); sign column + u64 pair + column read 4831 (21.1) | 5386 | 0.47x -> 1.11x | - | - | | iq2xxs | 11061 (48.2); sign column + u64 pair 5121-5487 (noise band) | 5124 | 0.46x -> 0.93-1.00x | - | - | -The full ladder (`harness/kernel_ladder.sh`, 2026-09-01, zen2 one thread, the stamped seat per format, -best of 5 interleaved rounds; ratio = reference / ours, >= 1.00 = ours faster): +The zen2 ladder after the k3/k2 step and the bench fixes (64-byte-aligned planes, normal scale bytes; +`harness/kernel_ladder.sh`, 2026-09-01, one thread, the stamped seat per format, best of 5 interleaved +rounds; ratio = reference / ours, >= 1.00 = ours faster): + +| format | gemv ours us | gemv ref us | ratio | tile ours us | tile ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 3179 | 4325.63 | 1.36 | 576089 | 923211.00 | 1.60 | +| k4 | 1948 | 2391.08 | 1.23 | 547996 | 821706.50 | 1.50 | +| k5 | 3700 | 3264.03 | 0.88 | 583516 | 1311611.00 | 2.25 | +| k6 | 4204 | 3647.44 | 0.87 | 679148 | 1083505.00 | 1.60 | +| q40 | 1865 | 2839.84 | 1.52 | 387002 | 1018999.00 | 2.63 | +| q51 | 2824 | 4152.78 | 1.47 | 677326 | 1915920.00 | 2.83 | +| iq4xs | 2101 | 3357.24 | 1.60 | 671645 | 1645997.00 | 2.45 | +| k3 | 3092 | 2630.86 | 0.85 | 766074 | 1284702.00 | 1.68 | +| iq3s | 7668 | 10103.66 | 1.32 | 673687 | 5115105.00 | 7.59 | +| iq3xxs | 6601 | 6547.02 | 0.99 | 679775 | 3281595.00 | 4.83 | +| iq4nl | 2208 | 3027.06 | 1.37 | 592394 | 1125061.00 | 1.90 | +| k2 | 1370 | 1973.20 | 1.44 | 692253 | 770237.50 | 1.11 | +| iq2s | 5052 | 5069.15 | 1.00 | 711298 | 2513290.00 | 3.53 | +| iq2xs | 4506 | 5265.71 | 1.17 | 724580 | 2738442.00 | 3.78 | +| iq2xxs | 4859 | 5119.28 | 1.05 | 659785 | 2542052.00 | 3.85 | +| mx4 | 2205 | 3009.07 | 1.36 | 623427 | 1421391.00 | 2.28 | + +k3 decode is the one row that disagrees with its own history (2445 us after its fix, 3092 here) - a +run-to-run spread like Intel's bimodal k6, under investigation. + +The first zen2 ladder (before the k3/k2 step, unaligned planes, random scales): | format | gemv ours us | gemv ref us | ratio | tile ours us | tile ref us | ratio | |---|---|---|---|---|---|---| From 4e817275f71866c0cd581766137263eac3a95424 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 04:28:45 -0700 Subject: [PATCH 026/123] kq_kernel_bench: one arena for every plane at fixed staggered offsets - separate arrays landed at run-dependent relative addresses and k6 decode read 3.9-6.0 ms across identical runs; now 4.0-4.4 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../benchmarks/matmul/kq_kernel_bench.das | 141 +++++++++++------- 1 file changed, 83 insertions(+), 58 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index feed9612ca..fccc5ad4e3 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -56,7 +56,6 @@ typedef LayoutFn = function<() : int> let TILE_TOKENS = 4l // tokens one kq / q51 / mx4 tile call covers (q8's tokstep companion says its own) let MX4_NAME = "mx4" // mxfp4 has no KqFmt row; it rides the q8q8 grid as a companion -let SCALE_BYTE = uint8(0x3C) // as f16 0x3C3C = 1.06, as f32 0x3C3C3C3C = 0.0115, as e8m0 2^-67, as a 6-bit sub-scale 60 - normal in every scale form def fmt_of_name(name : string) : KqFmt { for (f in type) { @@ -156,55 +155,81 @@ def plane_shape(f : KqFmt; mx4 : bool; n : int64) : PlaneShape { return PlaneShape(wq_row = nsb * kq_qsb(f), tq_row = nsb * tile_kq_bytes_per_sb(f), ws_row = nsb * kq_ssb(f)) } +let ALIGN = 64l // the image's planes are page-aligned (ARCHITECTURE_IMAGE.md, image-page-alignment); a 64-byte load split across cache lines is what Intel penalizes 2x +let STAGGER = 1088l // plane starts differ in their low 12 address bits (1088 = 1024 + 64 keeps six starts distinct mod 4096): planes that alias in the L1/L2 set logic make a run's time depend on where the heap put them +let SCALE_BYTE = uint8(0x3C) // as f16 0x3C3C = 1.06, as f32 0x3C3C3C3C = 0.0115, as e8m0 2^-67, as a 6-bit sub-scale 60 - normal in every scale form + +//! every plane of one format in ONE arena at fixed offsets: the heap places separate arrays at +//! run-dependent relative addresses, and k6 decode read 3.9-6.0 ms across identical runs from that alone struct Planes { - kq : array - tkq : array // the tile arm's weight plane (the unpacked panel form where the format has one) - ks : array - xq : array - xs : array - xbs : array - y : array + arena : array + kq : int64 // the gemv's weight plane + tkq : int64 // the tile's weight plane (the unpacked panel form where the format has one) + ks : int64 // scale plane + xq : int64 // int8 activations + xs : int64 // f32 activation scales + xbs : int64 // int per-16 activation sums + y : int64 // f32 outputs } -let ALIGN = 64l // the image's planes are page-aligned (ARCHITECTURE_IMAGE.md, image-page-alignment); a 64-byte load split across cache lines is what Intel penalizes 2x - -def sized(var a : array; n : int64) { - a |> reserve(n + ALIGN) - a |> resize(n + ALIGN) +//! the next plane start: 64-byte aligned, `slot` staggers past the running end +def next_plane(var end : int64&; bytes : int64; slot : int) : int64 { + let start = ((end + ALIGN - 1l) / ALIGN) * ALIGN + int64(slot) * STAGGER + end = start + bytes + return start } -//! the first 64-byte-aligned element of an over-allocated plane -def aligned_start(p : void?) : int64 => (ALIGN - int64(intptr(p) & uint64(ALIGN - 1l))) & (ALIGN - 1l) +//! the arena's first 64-byte-aligned byte +def arena_base(var p : Planes) : int64 { + unsafe { + return (ALIGN - int64(intptr(addr(p.arena[0])) & uint64(ALIGN - 1l))) & (ALIGN - 1l) + } +} def fill_planes(var p : Planes; sh : PlaneShape; n, d, ntok : int64) { - p.kq |> sized(d * sh.wq_row) - p.tkq |> sized(d * sh.tq_row) - p.ks |> sized(d * sh.ws_row) - p.xq |> sized(ntok * n) - p.xs |> sized(ntok * n / 32l) - p.xbs |> sized(ntok * n / 16l) - p.y |> sized(ntok * d) + var end = 0l + p.kq = next_plane(end, d * sh.wq_row, 0) + p.tkq = next_plane(end, d * sh.tq_row, 1) + p.ks = next_plane(end, d * sh.ws_row, 2) + p.xq = next_plane(end, ntok * n, 3) + p.xs = next_plane(end, ntok * n / 32l * 4l, 4) + p.xbs = next_plane(end, ntok * n / 16l * 4l, 5) + p.y = next_plane(end, ntok * d * 4l, 6) + p.arena |> reserve(end + ALIGN) + p.arena |> resize(end + ALIGN) + let base = arena_base(p) + p.kq += base + p.tkq += base + p.ks += base + p.xq += base + p.xs += base + p.xbs += base + p.y += base var seed = 0x1234u - for (b in p.kq) { + for (i in range64(p.kq, p.kq + d * sh.wq_row)) { seed = seed * 1664525u + 1013904223u - b = uint8(seed >> 24u) + p.arena[i] = uint8(seed >> 24u) } - for (b in p.tkq) { + for (i in range64(p.tkq, p.tkq + d * sh.tq_row)) { seed = seed * 1664525u + 1013904223u - b = uint8(seed >> 24u) + p.arena[i] = uint8(seed >> 24u) } - for (b in p.ks) { - b = SCALE_BYTE // scale planes hold normal numbers in every scale form, never random bytes: Intel runs denormal math ~100x slower, AMD does not, and a random byte pattern is a denormal or an infinity often enough to bend a table + for (i in range64(p.ks, p.ks + d * sh.ws_row)) { + p.arena[i] = SCALE_BYTE // scale planes hold normal numbers in every scale form, never random bytes: Intel runs denormal math ~100x slower } - for (q in p.xq) { + for (i in range64(p.xq, p.xq + ntok * n)) { seed = seed * 1664525u + 1013904223u - q = int8(seed >> 24u) - } - for (s in p.xs) { - s = 0.01 + p.arena[i] = uint8(seed >> 24u) } - for (bsum in p.xbs) { - bsum = 128 + unsafe { + var xs = addr(p.arena[p.xs]) + for (i in range64(ntok * n / 32l)) { + xs[i] = 0.01 + } + var xbs = addr(p.arena[p.xbs]) + for (i in range64(ntok * n / 16l)) { + xbs[i] = 128 + } } } @@ -269,10 +294,10 @@ def time_rows(var stats : array; rounds : int; blk : block<(row : int) } } -def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; var p : Planes; nsb : int64) { +def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; var p : Planes; nsb : int64; sh : PlaneShape) { let n = int64(cfg.n) let d = int64(cfg.d) - let plane_bytes = long_length(p.kq) + long_length(p.ks) - 2l * ALIGN + let plane_bytes = int64(cfg.d) * (sh.wq_row + sh.ws_row) var vs <- gemv_variants(f) var stats : array var fns : array @@ -283,12 +308,12 @@ def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va } } unsafe { - let kqp = addr(p.kq[aligned_start(addr(p.kq[0]))]) - let ksp = addr(p.ks[aligned_start(addr(p.ks[0]))]) - let xqp = addr(p.xq[aligned_start(addr(p.xq[0]))]) - let xsp = addr(p.xs[aligned_start(addr(p.xs[0])) / 4l]) - let xbsp = addr(p.xbs[aligned_start(addr(p.xbs[0])) / 4l]) - var yp = addr(p.y[aligned_start(addr(p.y[0])) / 4l]) + let kqp = addr(p.arena[p.kq]) + let ksp = addr(p.arena[p.ks]) + let xqp = addr(p.arena[p.xq]) + let xsp = addr(p.arena[p.xs]) + let xbsp = addr(p.arena[p.xbs]) + var yp = addr(p.arena[p.y]) time_rows(stats, cfg.rounds) $(row : int) { invoke(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) } @@ -307,7 +332,7 @@ def bench_tile_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va let ntok = int64(cfg.ntok) let qrow = sh.tq_row let srow = sh.ws_row - let plane_bytes = long_length(p.tkq) + long_length(p.ks) - 2l * ALIGN + let plane_bytes = int64(cfg.d) * (sh.tq_row + sh.ws_row) var mrs <- companion_ints(layout_variants(f)) var vs <- tile_variants(f) var stats : array @@ -327,12 +352,12 @@ def bench_tile_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va mrOf |> push(mr) } unsafe { - let kqp = addr(p.tkq[aligned_start(addr(p.tkq[0]))]) - let ksp = addr(p.ks[aligned_start(addr(p.ks[0]))]) - let xqp = addr(p.xq[aligned_start(addr(p.xq[0]))]) - let xsp = addr(p.xs[aligned_start(addr(p.xs[0])) / 4l]) - let xbsp = addr(p.xbs[aligned_start(addr(p.xbs[0])) / 4l]) - var yp = addr(p.y[aligned_start(addr(p.y[0])) / 4l]) + let kqp = addr(p.arena[p.tkq]) + let ksp = addr(p.arena[p.ks]) + let xqp = addr(p.arena[p.xq]) + let xsp = addr(p.arena[p.xs]) + let xbsp = addr(p.arena[p.xbs]) + var yp = addr(p.arena[p.y]) time_rows(stats, cfg.rounds) $(row : int) { let mr = mrOf[row] for (g in range64(d / mr)) { @@ -416,19 +441,19 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes let d = int64(cfg.d) let ntok = int64(cfg.ntok) let name = mx4 ? MX4_NAME : "q8" - let plane_bytes = long_length(p.kq) + long_length(p.ks) - 2l * ALIGN + let plane_bytes = int64(cfg.d) * (sh.wq_row + sh.ws_row) var wvs <- q8q8_family_live_variants() // the family's witness rows: the amx one takes the per-process tile permission its tiles need (SIGILL without it), as the probe does for (w in wvs) { invoke(w._1) } delete wvs unsafe { - let kqp = addr(p.kq[aligned_start(addr(p.kq[0]))]) - let ksp = addr(p.ks[aligned_start(addr(p.ks[0]))]) - let xqp = addr(p.xq[aligned_start(addr(p.xq[0]))]) - let xsp = addr(p.xs[aligned_start(addr(p.xs[0])) / 4l]) - let xbsp = addr(p.xbs[aligned_start(addr(p.xbs[0])) / 4l]) - var yp = addr(p.y[aligned_start(addr(p.y[0])) / 4l]) + let kqp = addr(p.arena[p.kq]) + let ksp = addr(p.arena[p.ks]) + let xqp = addr(p.arena[p.xq]) + let xsp = addr(p.arena[p.xs]) + let xbsp = addr(p.arena[p.xbs]) + var yp = addr(p.arena[p.y]) if (!cfg.no_gemv) { var stats : array var fnsQ : array @@ -533,7 +558,7 @@ def main : int { bench_q8_family(rows, cfg, fe._2, p, nsb, sh) } else { if (!cfg.no_gemv) { - bench_gemv_kq(rows, cfg, fe._0, fe._1, p, nsb) + bench_gemv_kq(rows, cfg, fe._0, fe._1, p, nsb, sh) } if (cfg.ntok > 0) { bench_tile_kq(rows, cfg, fe._0, fe._1, p, nsb, sh) From 37bdeaa58ee940ebe93c982c65097577035cc366 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 04:33:18 -0700 Subject: [PATCH 027/123] kernel parity, ARM grid decode: row pairs under the sdot lattice - two rows x 8 weights per vector straight from the grid words The decode shape of the five grid formats on the NEON lattice composes each <16 x i8> weight vector as [rowA 8 weights | rowB 8 weights] from the existing per-dword decodes (one u64 grid entry per row for the iq2 formats), negates it with the sign column broadcast per row half, replicates the 8 activation bytes (ld1r) and feeds the non-indexed sdot; one pairwise add per row quad at block end yields the [A, B, C, D] accumulator the fold already takes - no panel, no per-dword sign math. Gated on DOT_SDOT, one token and width 128, so x86 emission is byte-identical. M1 one thread, us, before -> after (reference): iq2s 4932 -> 3256 (4917), iq3s 6089 -> 4937 (5682), iq2xs 6375 -> 4244 (3274), iq2xxs 6063 -> 4085 (3437), iq3xxs 6735 -> 5235 (4867). gen_tune_probe TEST 65/65 on the M1. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 88 ++++++++++++++++++- plans/kernel_parity_pass.md | 6 ++ 2 files changed, 92 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 35a96101e8..913802bfde 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1561,6 +1561,83 @@ def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { +//! the 8 activation bytes at byte offset `off` of token 0, replicated into both 64-bit lanes +//! (ld1r {v.2d} after lowering): the non-indexed sdot's operand for a [rowA 8w | rowB 8w] weight vector +def private bcast_qword_x(var te : TileEmit; var off : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { + let b = te.builder + var p = LLVMBuildGEP2(b, te.types.t_int8, te.x[0], off, "") + var q = LLVMBuildLoad2Aligned(b, te.types.t_int64, p, 1u, "") + var v2 = LLVMVectorType(te.types.t_int64, 2u) + var v = LLVMBuildInsertElement(b, LLVMGetUndef(v2), q, LLVMConstInt(te.types.t_int32, 0ul, 0), "") + var spl = LLVMBuildShuffleVector(b, te.types, v, v, [0, 0], "") + return LLVMBuildBitCast(b, spl, te.v16i8, name) +} + +//! row `r`'s 8 raw grid weights for (blk, l) as <8 x i8> - the two dword decodes side by side +def private compose_row8(var te : TileEmit; var wbP : LLVMOpaqueValue?; r, blk, l : int) : LLVMOpaqueValue? { + let b = te.builder + var base = LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(r * 4)), "") + var w0 = decode_kq_dword(te, base, blk, l, 0) + var w1 = decode_kq_dword(te, base, blk, l, 1) + var v2 = LLVMVectorType(te.types.t_int32, 2u) + var v = LLVMBuildInsertElement(b, LLVMGetUndef(v2), w0, LLVMConstInt(te.types.t_int32, 0ul, 0), "") + v = LLVMBuildInsertElement(b, v, w1, LLVMConstInt(te.types.t_int32, 1ul, 0), "") + return LLVMBuildBitCast(b, v, LLVMVectorType(te.types.t_int8, 8u), "") +} + +//! the row pair's sign mask off the sign column (byte r*4 + l per row): lanes 0-7 take row rA's byte, +//! 8-15 row rB's, bit k of the byte selects lane k - then (w ^ m) - m negates the masked weights +def private negate_pair(var te : TileEmit; var w, col : LLVMOpaqueValue?; rA, rB, l : int; name : string) : LLVMOpaqueValue? { + let b = te.builder + let idx <- [for (i in range(16)); (i < 8 ? rA : rB) * 4 + l] + var bytes = LLVMBuildShuffleVector(b, te.types, col, col, idx, "sgp") + var bits : array + bits |> reserve(16) + for (i in range(16)) { + bits |> push(LLVMConstInt(te.types.t_int8, uint64(1 << (i % 8)), 0)) + } + var bitsv = LLVMConstVector(array_data_ptr(bits), 16u) + var m = LLVMBuildSExt(b, LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntEQ, LLVMBuildAnd(b, bytes, bitsv, ""), bitsv, ""), te.v16i8, "sgpm") + return LLVMBuildSub(b, LLVMBuildXor(b, w, m, ""), m, name) +} + +//! One block of a grid format's decode under the sdot lattice as row PAIRS: a <16 x i8> weight vector is +//! two rows x 8 weights (one u64 grid entry per row for iq2), the activation's 8 bytes replicated, the +//! non-indexed sdot's lanes [A lo, A hi, B lo, B hi]; one pairwise add per row quad yields the fold's [A, B, C, D]. +def private emit_grid_pairs_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var signCol : LLVMOpaqueValue? [2]; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { + let b = te.builder + for (qd in range(te.rq)) { + var acc : LLVMOpaqueValue? [2] // per row pair; the lo strip (l < 2) when iq2 + var acc1 : LLVMOpaqueValue? [2] // iq2: the hi strip (l >= 2) + for (pr in range(2)) { + acc[pr] = LLVMConstNull(te.vni32) + acc1[pr] = LLVMConstNull(te.vni32) + } + for (l in range(4)) { + var xv = bcast_qword_x(te, LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32 + l * 8)), ""), "x8{blk}_{l}") + for (pr in range(2)) { + let rA = qd * te.rv + 2 * pr + let rB = rA + 1 + var gA = compose_row8(te, wbP, rA, blk, l) + var gB = compose_row8(te, wbP, rB, blk, l) + var w = LLVMBuildShuffleVector(b, te.types, gA, gB, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], "wp{blk}_{l}_{qd}_{pr}") + var ws = negate_pair(te, w, signCol[qd], 2 * pr, 2 * pr + 1, l, "wps{blk}_{l}_{qd}_{pr}") + var args <- [(iq2 && l >= 2) ? acc1[pr] : acc[pr], ws, xv] + var d = LLVMBuildCall2(b, te.sdot_ty, te.sdot_decl, args, "sdotp") + if (iq2 && l >= 2) { + acc1[pr] = d + } else { + acc[pr] = d + } + } + } + a[qd] = LLVMBuildAdd(b, a[qd], pairwise_add_i32(te, acc[0], acc[1], "ap{blk}_{qd}"), "") + if (iq2) { + a1[qd] = LLVMBuildAdd(b, a1[qd], pairwise_add_i32(te, acc1[0], acc1[1], "ap1{blk}_{qd}"), "") + } + } +} + // One 256-weight SUPERBLOCK, IQ4_XS/iq3s grp form (te.kq = 44/33): mx4's LUT decode (44) // or the byte-expanded signed panel (33; the gemv gathers it per superblock, the tile gets the // runtime unpack) — dot_lane's sign-trick lattice, k4's fold with no min term. @@ -1573,12 +1650,16 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let iq2 = te.kq == 23 || te.kq == 24 // iq2s/iq2xs: per-16 UNSIGNED strips - lo/hi halves keep split accumulators let nl = te.kq == 45 // iq4nl: the 44 LUT decode with q40's PER-BLOCK f16 d fold (no sub-scales, no min term) let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first + // the sdot lattice's decode shape composes row PAIRS straight from the grid words (emit_grid_pairs_blk) - no panel + let pairs = gather && te.dotKind == DOT_SDOT && tokCount == 1 && te.rv == 4 var wbase = te.wg var wb : LLVMOpaqueValue? var wbP : LLVMOpaqueValue? if (gather) { wbP = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr) * uint64(kq_gather_stride(te.kq))), "kq.wb") - emit_kq_gather(te, sbi) + if (!pairs) { + emit_kq_gather(te, sbi) + } wbase = te.iq3s_panel wb = te.types->ConstI64(0ul) } else { @@ -1626,7 +1707,10 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var signCol[qd] = kq_sign_bytes(te, wbP, blk, qd, "sgc{blk}_{qd}") } } - for (j in range(4)) { + if (pairs) { + emit_grid_pairs_blk(te, wbP, xb, blk, signCol, a, a1, iq2) + } + for (j in range(pairs ? 0 : 4)) { for (qd in range(rq)) { var noff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 16 + j * 4) * mr + qd * w8)), "") var wlo : LLVMOpaqueValue? diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 54aef00f4e..5a7b5fccc9 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -227,6 +227,12 @@ Model: iq2xs ~2570 us = 1.27x. vqtbl gathers are impossible (1-2 KB grids vs tbl masked negate already beats llama.cpp's own mask expansion. Fallback if deferred: plane-carried sign bytes (size-neutral for iq2xxs/iq3xxs, +6% for iq2xs; touches the plane, both ISAs) -> 0.72-0.98x only. Also from the memo: the column-read-vs-byte-read decision was made on x86 and should be a perm seat. +LANDED 2026-09-01 (the row-pair decode under the sdot lattice, gated DOT_SDOT + decode shape + width +128; x86 untouched): M1 one thread, us, before -> after (reference): iq2s 4932 -> 3256 (4917, 1.51x), +iq3s 6089 -> 4937 (5682, 1.15x), iq2xs 6375 -> 4244 (3274, 0.77x), iq2xxs 6063 -> 4085 (3437, 0.84x), +iq3xxs 6735 -> 5235 (4867, 0.93x); TEST 65/65 on the M1. Still behind on the three ksigns formats: +the sign mask is five NEON ops per pair per 8-group where a +-1 table (keven_signs form) is two loads +and one multiply - the next ARM step; then the per-dword scalar index math. Bench on AMX boxes: the tune-mode q8 tile SIGILL (above) - read how gen_tune_probe reaches the grant (q8q8_family_live_variants + the amx cfg companion?) and do the same; k6 bimodality on Granite Rapids. From 7ddb256f8370f2eb54258b0d14db19d2f96a99ce Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 04:49:38 -0700 Subject: [PATCH 028/123] kernel parity, ARM grid decode: the ksigns formats sign through a +-1 table row per code, the iq2 formats load the u64 grid entry whole The row-pair decode's sign step splits by where the signs live: iq3s and iq2s keep the mask off the plane's sign column (one 16-byte load per row quad, four NEON ops per pair), the three ksigns formats read one [128 x i64] table row per 7-bit code - the code's parity folded into the table, no per-lane parity arithmetic - and multiply. The iq2 formats' grid entry loads as one i64 straight into the <8 x i8> row. M1 one thread, us, before -> after (reference): iq2xs 4244 -> 2636 (3274), iq2xxs 4085 -> 2873 (3437), iq3xxs 5235 -> 3576 (4867), iq2s 3256 -> 3260 (4917), iq3s 4937 -> 4929 (5682); gen_tune_probe TEST 90 ok. The table for the plane-byte formats measured 4167 / 5608 - four scalar loads per pair against one column load, and the M1 sustains three loads to four NEON ops per cycle - so those two keep the mask. kernel_ladder.sh refuses a reference binary without the GGML_BENCH_THREADS define: such a binary runs every core and the ratio column lies. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 168 ++++++++++++------ modules/dasLLAMA/harness/kernel_ladder.sh | 1 + plans/kernel_parity_pass.md | 42 ++++- 3 files changed, 154 insertions(+), 57 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 913802bfde..40915ab5a7 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -247,6 +247,7 @@ struct private TileEmit { psign_decl : LLVMOpaqueValue? // x64 width-256 sign apply (VPSIGNB) iq3s_panel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) + sign_code_pm1_g : LLVMOpaqueValue? // the sdot row-pair decode: ksigns code -> eight +-1 bytes, [128 x i64], parity bit folded in gfn : LLVMOpaqueValue? // the function under emit (block creation for emitted inner loops) gctx : LLVMOpaqueContext? psign_ty : LLVMOpaqueType? @@ -1323,11 +1324,17 @@ def private grid_word(var te : TileEmit; var gidx : LLVMOpaqueValue?) : LLVMOpaq //! (the h = 0 and h = 1 decodes share the load through CSE) def private grid_pair_half(var te : TileEmit; var idx2 : LLVMOpaqueValue?; h : int) : LLVMOpaqueValue? { let b = te.builder - var gp = LLVMBuildGEP2(b, te.types.t_int64, te.iq3s_grid_g, LLVMBuildLShr(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), ""), "") - var g64 = LLVMBuildLoad2Aligned(b, te.types.t_int64, gp, 4u, "") + var g64 = grid_pair_i64(te, idx2) return LLVMBuildTrunc(b, h == 0 ? g64 : LLVMBuildLShr(b, g64, LLVMConstInt(te.types.t_int64, 32ul, 0), ""), te.types.t_int32, "") } +//! the u64 grid's word pair at even index `idx2` as one i64 - a row's 8 weights in one load +def private grid_pair_i64(var te : TileEmit; var idx2 : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + var gp = LLVMBuildGEP2(b, te.types.t_int64, te.iq3s_grid_g, LLVMBuildLShr(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), ""), "") + return LLVMBuildLoad2Aligned(b, te.types.t_int64, gp, 4u, "") +} + //! iq3s: 8-bit qs byte + the block's qh bit = the 9-bit grid index def private decode_iq3s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { let b = te.builder @@ -1343,33 +1350,94 @@ def private decode_iq3xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; return grid_word(te, wg_col_byte(te, base, blk * 8 + 2 * l + h)) } -//! iq2s: qs byte | the qh 0x300 window = the 10-bit index, doubled into the u64 grid's word pair -def private decode_iq2s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { - let b = te.builder - let mr = te.interleave - var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) - var iv0 = wg_byte(te, base, wg_col(mr, blk * 4 + l)) - var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 0x300ul, 0), "") - var idx2 = LLVMBuildShl(b, LLVMBuildOr(b, iv0, hb, ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") - return grid_pair_half(te, idx2, h) -} - -//! iq2xs: the u16 word's low 9 bits index the u64 grid's word pair (the column read measured 6501 -> 4702 us; one u16 load measured slower than two bytes) -def private decode_iq2xs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { +//! iq2xs: row `base`'s u16 word for (blk, l) - low 9 bits the grid index, the top 7 the ksigns code +//! (the column read measured 6501 -> 4702 us; one u16 load measured slower than two bytes) +def private iq2xs_word16(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l : int) : LLVMOpaqueValue? { let b = te.builder let lp = (blk * 4 + l) * 2 var qlo = wg_col_byte(te, base, lp) var qhi = wg_col_byte(te, base, lp + 1) - var w16 = LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "") - var idx2 = LLVMBuildShl(b, LLVMBuildAnd(b, w16, LLVMConstInt(te.types.t_int32, 511ul, 0), ""), LLVMConstInt(te.types.t_int32, 1ul, 0), "") - return grid_pair_half(te, idx2, h) + return LLVMBuildOr(b, qlo, LLVMBuildShl(b, qhi, LLVMConstInt(te.types.t_int32, 8ul, 0), ""), "") } -//! iq2xxs: block b's grid bytes (column 2b) index the u64 grid's word pair -def private decode_iq2xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { +//! the iq2 formats' doubled index into the u64 grid's word pairs for row `base`, (blk, l): iq2s the qs byte +//! under the qh 0x300 window, iq2xs the u16 word's low 9 bits, iq2xxs block b's grid byte (column 2b) +def private iq2_idx2(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l : int) : LLVMOpaqueValue? { let b = te.builder - var idx2 = LLVMBuildShl(b, wg_byte(te, base, 2 * blk * te.interleave * 4 + l), LLVMConstInt(te.types.t_int32, 1ul, 0), "") - return grid_pair_half(te, idx2, h) + var idx : LLVMOpaqueValue? + if (te.kq == 23) { + let mr = te.interleave + var qh = wg_byte(te, base, wg_col(mr, 64 + blk)) + var hb = LLVMBuildAnd(b, LLVMBuildShl(b, qh, LLVMConstInt(te.types.t_int32, uint64(8 - 2 * l), 0), ""), LLVMConstInt(te.types.t_int32, 0x300ul, 0), "") + idx = LLVMBuildOr(b, wg_byte(te, base, wg_col(mr, blk * 4 + l)), hb, "") + } elif (te.kq == 24) { + idx = LLVMBuildAnd(b, iq2xs_word16(te, base, blk, l), LLVMConstInt(te.types.t_int32, 511ul, 0), "") + } else { + idx = wg_byte(te, base, 2 * blk * te.interleave * 4 + l) + } + return LLVMBuildShl(b, idx, LLVMConstInt(te.types.t_int32, 1ul, 0), "") +} + +//! row `base`'s 7-bit ksigns code for (blk, l) - iq2xs the u16 word's top bits, iq2xxs/iq3xxs the aux32's 7-bit fields +def private kq_row_sign_code(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l : int) : LLVMOpaqueValue? { + let b = te.builder + if (te.kq == 24) return LLVMBuildLShr(b, iq2xs_word16(te, base, blk, l), LLVMConstInt(te.types.t_int32, 9ul, 0), "") + var aux = wg_i32(te, base, (te.kq == 34 ? 16 + blk : 2 * blk + 1) * te.interleave * 4) + var sh = l == 0 ? aux : LLVMBuildLShr(b, aux, LLVMConstInt(te.types.t_int32, uint64(7 * l), 0), "") + return LLVMBuildAnd(b, sh, LLVMConstInt(te.types.t_int32, 127ul, 0), "") +} + +//! the eight +-1 bytes of a ksigns code: one table row (bit k of the expanded byte set = lane k negative) +def private sign_pm1_row(var te : TileEmit; var code : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + var gp = LLVMBuildGEP2(b, te.types.t_int64, te.sign_code_pm1_g, code, "") + return LLVMBuildBitCast(b, LLVMBuildLoad2Aligned(b, te.types.t_int64, gp, 8u, ""), LLVMVectorType(te.types.t_int8, 8u), "") +} + +//! the row pair's sign mask off the plane's sign column (byte r*4 + l per row): lanes 0-7 take row rA's +//! byte, 8-15 row rB's, bit k selects lane k - then (w ^ m) - m negates the masked weights +def private negate_pair(var te : TileEmit; var w, col : LLVMOpaqueValue?; rA, rB, l : int; name : string) : LLVMOpaqueValue? { + let b = te.builder + let idx <- [for (i in range(16)); (i < 8 ? rA : rB) * 4 + l] + var bytes = LLVMBuildShuffleVector(b, te.types, col, col, idx, "sgp") + var bits : array + bits |> reserve(16) + for (i in range(16)) { + bits |> push(LLVMConstInt(te.types.t_int8, uint64(1 << (i % 8)), 0)) + } + var bitsv = LLVMConstVector(array_data_ptr(bits), 16u) + var m = LLVMBuildSExt(b, LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntEQ, LLVMBuildAnd(b, bytes, bitsv, ""), bitsv, ""), te.v16i8, "sgpm") + return LLVMBuildSub(b, LLVMBuildXor(b, w, m, ""), m, name) +} + +//! eight +-1 bytes for sign byte `sb`, lane k negative when bit k is set +def private pm1_of(sb : int) : uint64 { + var v = 0ul + for (k in range(8)) { + v |= uint64(((sb >> k) & 1) != 0 ? 0xFF : 0x01) << uint64(8 * k) + } + return v +} + +//! the row-pair decode's sign table: the 128 ksigns codes with the code's parity as the 8th bit +//! (ksigns[i] == i | parity(i) << 7), each as its eight +-1 bytes +def private sign_pm1_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + if (te.dotKind != DOT_SDOT) return // only the sdot lattice's row-pair decode reads it + te.sign_code_pm1_g = LLVMGetNamedGlobal(mod, "dasllama.sign.code_pm1") + if (te.sign_code_pm1_g != null) return + var codes : array + codes |> reserve(128) + for (c in range(128)) { + var par = c ^ (c >> 4) + par ^= par >> 2 + par ^= par >> 1 + codes |> push(LLVMConstInt(te.types.t_int64, pm1_of(c | ((par & 1) << 7)), 0)) + } + te.sign_code_pm1_g = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int64, 128u), "dasllama.sign.code_pm1") + LLVMSetInitializer(te.sign_code_pm1_g, LLVMConstArray(te.types.t_int64, array_data_ptr(codes), 128u)) + LLVMSetGlobalConstant(te.sign_code_pm1_g, 1) + LLVMSetLinkage(te.sign_code_pm1_g, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(te.sign_code_pm1_g, 64u) } //! block `blk`'s sign bytes for rows qd*rv.., byte (r, l) at r*4 + l - the plane's own column for @@ -1431,9 +1499,7 @@ def private apply_sign_col(var te : TileEmit; var w, col : LLVMOpaqueValue?; l, def private decode_kq_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { if (te.kq == 33) return decode_iq3s_dword(te, base, blk, l, h) if (te.kq == 34) return decode_iq3xxs_dword(te, base, blk, l, h) - if (te.kq == 23) return decode_iq2s_dword(te, base, blk, l, h) - if (te.kq == 24) return decode_iq2xs_dword(te, base, blk, l, h) - return decode_iq2xxs_dword(te, base, blk, l, h) + return grid_pair_half(te, iq2_idx2(te, base, blk, l), h) } //! bytes per row per superblock of a panel format's packed grp region @@ -1573,39 +1639,27 @@ def private bcast_qword_x(var te : TileEmit; var off : LLVMOpaqueValue?; name : return LLVMBuildBitCast(b, spl, te.v16i8, name) } -//! row `r`'s 8 raw grid weights for (blk, l) as <8 x i8> - the two dword decodes side by side -def private compose_row8(var te : TileEmit; var wbP : LLVMOpaqueValue?; r, blk, l : int) : LLVMOpaqueValue? { +//! row `base`'s 8 raw grid weights for (blk, l) as <8 x i8>: the iq2 formats' u64 grid entry in one +//! load, iq3s/iq3xxs the two dword decodes side by side +def private row_weights8(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l : int) : LLVMOpaqueValue? { let b = te.builder - var base = LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(r * 4)), "") + var v8i8 = LLVMVectorType(te.types.t_int8, 8u) + if (te.kq != 33 && te.kq != 34) return LLVMBuildBitCast(b, grid_pair_i64(te, iq2_idx2(te, base, blk, l)), v8i8, "") var w0 = decode_kq_dword(te, base, blk, l, 0) var w1 = decode_kq_dword(te, base, blk, l, 1) var v2 = LLVMVectorType(te.types.t_int32, 2u) var v = LLVMBuildInsertElement(b, LLVMGetUndef(v2), w0, LLVMConstInt(te.types.t_int32, 0ul, 0), "") v = LLVMBuildInsertElement(b, v, w1, LLVMConstInt(te.types.t_int32, 1ul, 0), "") - return LLVMBuildBitCast(b, v, LLVMVectorType(te.types.t_int8, 8u), "") + return LLVMBuildBitCast(b, v, v8i8, "") } -//! the row pair's sign mask off the sign column (byte r*4 + l per row): lanes 0-7 take row rA's byte, -//! 8-15 row rB's, bit k of the byte selects lane k - then (w ^ m) - m negates the masked weights -def private negate_pair(var te : TileEmit; var w, col : LLVMOpaqueValue?; rA, rB, l : int; name : string) : LLVMOpaqueValue? { - let b = te.builder - let idx <- [for (i in range(16)); (i < 8 ? rA : rB) * 4 + l] - var bytes = LLVMBuildShuffleVector(b, te.types, col, col, idx, "sgp") - var bits : array - bits |> reserve(16) - for (i in range(16)) { - bits |> push(LLVMConstInt(te.types.t_int8, uint64(1 << (i % 8)), 0)) - } - var bitsv = LLVMConstVector(array_data_ptr(bits), 16u) - var m = LLVMBuildSExt(b, LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntEQ, LLVMBuildAnd(b, bytes, bitsv, ""), bitsv, ""), te.v16i8, "sgpm") - return LLVMBuildSub(b, LLVMBuildXor(b, w, m, ""), m, name) -} - -//! One block of a grid format's decode under the sdot lattice as row PAIRS: a <16 x i8> weight vector is -//! two rows x 8 weights (one u64 grid entry per row for iq2), the activation's 8 bytes replicated, the -//! non-indexed sdot's lanes [A lo, A hi, B lo, B hi]; one pairwise add per row quad yields the fold's [A, B, C, D]. +//! One block of a grid format's decode under the sdot lattice as row PAIRS: a <16 x i8> weight vector is two +//! rows x 8 weights (one u64 grid entry per row for iq2), signed by a mask off the plane's sign column (iq3s/iq2s) +//! or the ksigns +-1 table rows; the activation's 8 bytes replicated; one pairwise add per row quad yields the fold's [A, B, C, D]. def private emit_grid_pairs_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var signCol : LLVMOpaqueValue? [2]; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { let b = te.builder + let concat <- [for (i in range(16)); i] + let colSigns = te.kq == 33 || te.kq == 23 for (qd in range(te.rq)) { var acc : LLVMOpaqueValue? [2] // per row pair; the lo strip (l < 2) when iq2 var acc1 : LLVMOpaqueValue? [2] // iq2: the hi strip (l >= 2) @@ -1617,11 +1671,16 @@ def private emit_grid_pairs_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue var xv = bcast_qword_x(te, LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32 + l * 8)), ""), "x8{blk}_{l}") for (pr in range(2)) { let rA = qd * te.rv + 2 * pr - let rB = rA + 1 - var gA = compose_row8(te, wbP, rA, blk, l) - var gB = compose_row8(te, wbP, rB, blk, l) - var w = LLVMBuildShuffleVector(b, te.types, gA, gB, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], "wp{blk}_{l}_{qd}_{pr}") - var ws = negate_pair(te, w, signCol[qd], 2 * pr, 2 * pr + 1, l, "wps{blk}_{l}_{qd}_{pr}") + var baseA = LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(rA * 4)), "") + var baseB = LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(rA * 4 + 4)), "") + var w = LLVMBuildShuffleVector(b, te.types, row_weights8(te, baseA, blk, l), row_weights8(te, baseB, blk, l), concat, "wp{blk}_{l}_{qd}_{pr}") + var ws : LLVMOpaqueValue? + if (colSigns) { + ws = negate_pair(te, w, signCol[qd], 2 * pr, 2 * pr + 1, l, "wps{blk}_{l}_{qd}_{pr}") + } else { + var sg = LLVMBuildShuffleVector(b, te.types, sign_pm1_row(te, kq_row_sign_code(te, baseA, blk, l)), sign_pm1_row(te, kq_row_sign_code(te, baseB, blk, l)), concat, "sp{blk}_{l}_{qd}_{pr}") + ws = LLVMBuildMul(b, w, sg, "wps{blk}_{l}_{qd}_{pr}") + } var args <- [(iq2 && l >= 2) ? acc1[pr] : acc[pr], ws, xv] var d = LLVMBuildCall2(b, te.sdot_ty, te.sdot_decl, args, "sdotp") if (iq2 && l >= 2) { @@ -1702,7 +1761,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var a1[i] = LLVMConstNull(te.vni32) } var signCol : LLVMOpaqueValue? [2] // this block's sign bytes, rows qd*rv.. at stride 4 - the gemv negates per vector, the tile's panel arrives signed - if (gather) { + if (gather && (!pairs || te.kq == 33 || te.kq == 23)) { for (qd in range(rq)) { signCol[qd] = kq_sign_bytes(te, wbP, blk, qd, "sgc{blk}_{qd}") } @@ -2920,6 +2979,7 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { } else { iq3xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } + sign_pm1_emit_globals(te, LLVMGetGlobalParent(gc.impl)) te.iq3s_panel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "iq3s.panel") LLVMSetAlignment(te.iq3s_panel, 32u) } diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh index a996014882..a8a89e44b7 100644 --- a/modules/dasLLAMA/harness/kernel_ladder.sh +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -39,6 +39,7 @@ echo "# kernel ladder $(date +%F) box=$(hostname) daslang=$DASLANG ref=$TBO # ---- ours: one process, every format, the stamped winner per format ---- [ -x "$TBO" ] || { echo "kernel_ladder: no test-backend-ops at '$TBO' (set LCPP_TBO)" >&2; exit 1; } +grep -q GGML_BENCH_THREADS "$TBO" || { echo "kernel_ladder: '$TBO' lacks the GGML_BENCH_THREADS define - it would run every core and the ratio column would lie (harness README)" >&2; exit 1; } set +e DAS_TUNE_MODE=normal DAS_JOBQUE_THREADS=1 "$DASLANG" -jit "$ROOT/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das" \ -- --fmt "$FMTS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv > "$WORK/ours.raw" 2> "$WORK/ours.err" diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 5a7b5fccc9..a876e25d2b 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -145,6 +145,36 @@ against 8.6-8.8 for k4/k2. A flat cost independent of the format is a shared mec decode problems. The model-level tg rows (0.70-0.92x) are this 2x kernel gap hidden behind memory bandwidth; on the 1B vehicles it shows, on a 27B it mostly does not - but the kernel is the same. +### zen2 v3 - the arena bench (2026-09-01, `harness/kernel_ladder.sh all`, one thread, best of 5 interleaved rounds) + +Every plane in ONE arena at fixed 64-byte-aligned staggered offsets; the reference is test-backend-ops at +6c84c7d5d with the GGML_BENCH_THREADS define. k6 read 5356 us here against 4.0-4.4 ms in the arena smoke - +its process-to-process swing is narrowed, not gone. + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 3142 / 4220 | 1.34 | 567862 / 907771 | 1.60 | +| k4 | 2005 / 2434 | 1.21 | 485803 / 846691 | 1.74 | +| k5 | 3467 / 3306 | 0.95 | 514648 / 1296856 | 2.52 | +| k6 | 5356 / 3683 | 0.69 | 634588 / 1076847 | 1.70 | +| q40 | 1901 / 2971 | 1.56 | 374388 / 1005240 | 2.69 | +| q51 | 2815 / 4107 | 1.46 | 662669 / 1830207 | 2.76 | +| iq4xs | 2076 / 3375 | 1.63 | 723720 / 1684220 | 2.33 | +| k3 | 2699 / 2700 | 1.00 | 733239 / 1245200 | 1.70 | +| iq3s | 7593 / 10345 | 1.36 | 743364 / 5194190 | 6.99 | +| iq3xxs | 6554 / 6524 | 1.00 | 739558 / 3293421 | 4.45 | +| iq4nl | 2126 / 3055 | 1.44 | 581628 / 1094468 | 1.88 | +| k2 | 1248 / 2158 | 1.73 | 636260 / 764748 | 1.20 | +| iq2s | 5044 / 5089 | 1.01 | 778057 / 2512394 | 3.23 | +| iq2xs | 4709 / 5320 | 1.13 | 799146 / 2692158 | 3.37 | +| iq2xxs | 4897 / 5086 | 1.04 | 738382 / 2537062 | 3.44 | +| mx4 | 2216 / 3113 | 1.41 | 618514 / 1441041 | 2.33 | + +Decode below 1.0: k6 0.69, k5 0.95; at 1.0: k3 1.00, iq3xxs 1.00, iq2s 1.01. Every tile ahead (1.20-6.99). + +M1 note: the ladder's reference must be `~/Work/llama.cpp/build/bin/test-backend-ops` (the fork checkout, +98c4764b6, carries the define); `build-cpu/` does not and ran every core - the ladder now refuses such a binary. + ## 3. Research memos (read before touching the kernels) - `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, @@ -230,9 +260,15 @@ Also from the memo: the column-read-vs-byte-read decision was made on x86 and sh LANDED 2026-09-01 (the row-pair decode under the sdot lattice, gated DOT_SDOT + decode shape + width 128; x86 untouched): M1 one thread, us, before -> after (reference): iq2s 4932 -> 3256 (4917, 1.51x), iq3s 6089 -> 4937 (5682, 1.15x), iq2xs 6375 -> 4244 (3274, 0.77x), iq2xxs 6063 -> 4085 (3437, 0.84x), -iq3xxs 6735 -> 5235 (4867, 0.93x); TEST 65/65 on the M1. Still behind on the three ksigns formats: -the sign mask is five NEON ops per pair per 8-group where a +-1 table (keven_signs form) is two loads -and one multiply - the next ARM step; then the per-dword scalar index math. +iq3xxs 6735 -> 5235 (4867, 0.93x); TEST 65/65 on the M1. +LANDED 2026-09-01, step two - signs and grid words: the ksigns formats take a +-1 table row per 7-bit +code (keven_signs form, [128 x i64]) and the iq2 formats load the u64 grid entry whole. M1 us, after +(reference): iq2xs 2636 (3274, 1.24x), iq2xxs 2873 (3437, 1.20x), iq3xxs 3576 (4867, 1.36x), iq2s 3260 +(4917, 1.51x), iq3s 4929 (5682, 1.15x); TEST 90 ok. All five grid decodes ahead on the M1. LESSON: the +table for the plane-byte formats (iq3s/iq2s) REGRESSED them (iq2s 3256 -> 4167, iq3s 4937 -> 5608) - +four scalar loads per pair replaced one 16-byte column load plus four NEON ops, and the M1 sustains 3 +loads against 4 NEON ops per cycle; those two keep the column mask. Count loads AND NEON ops per pair, +the bound is whichever is fuller. Bench on AMX boxes: the tune-mode q8 tile SIGILL (above) - read how gen_tune_probe reaches the grant (q8q8_family_live_variants + the amx cfg companion?) and do the same; k6 bimodality on Granite Rapids. From 9e61caf4139b8cacf81f57df0b0c9b69d7515dd1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 04:53:18 -0700 Subject: [PATCH 029/123] kernel parity ledgers: the ARM grid landing under followup 61 (the duplicate 61 renumbered 65), the x86 row-quad twin queued, the sidecar how-to checks the reference define --- modules/dasLLAMA/HOW_TO_GET_SIDECAR.md | 4 ++++ modules/dasLLAMA/followup_general.md | 21 +++++++++++++-------- plans/kernel_parity_pass.md | 11 +++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md index 1ec3378b52..a9adb4da38 100644 --- a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -88,6 +88,10 @@ cmake --build build-clean-cpu --target test-backend-ops llama-bench -j 16 `build-clean-cpu` is the flavor the tables name: no GPU backend, no BLAS - a GPU build's `llama-bench -ngl 0` is not CPU-only. +Confirm the define is in the binary the ladder gets: `grep -c GGML_BENCH_THREADS +build-clean-cpu/bin/test-backend-ops` prints 1. A box with several llama.cpp builds is where this +bites - a binary that prints 0 runs every core and the ratio column lies; `kernel_ladder.sh` refuses it. + ## 4. Correctness before any number ``` diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 822d7796bc..338bb1f0a4 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -707,14 +707,6 @@ the expected crowned entries' counts moved, once the cell picks a carrier whose shapes actually ready the tensor variants. -61. **An in-process image-off seam for facade-knob test cells.** A cell whose subject is a - facade lane knob (`load_asr_model` under `set_asr_tower_fp32`) must keep the facade - loader, and on a cold box that load BAKES the pinned lane's `.dlim` and GC-purges the - serving lane's flavor (`dlim_gc_stale`) - the class that broke `test_audio_embedder` - once already. The tests checklist ledgers the residue; the fix is an in-process - equivalent of `DASLLAMA_IMAGE=0` (`g_env_engine.image` is a `let` read at load), so such - cells can run image-free instead of risking the purge. - 57. **Plane types have no `long_length`.** `length(PlaneF)` / `length(PlaneU16)` return `int`, so every `uint64(length(t.blob) * 4l)` spelling caps a plane at 2^31 elements before the widening - headroom-only today (whisper large-v3's twin is ~632M elements). @@ -782,6 +774,12 @@ one gemv path (the tuner races the tile, where a gemv-only spelling ties). Kernel-level, one thread: iq3s 1.37x, iq2s 0.98x, iq2xxs 0.98x, iq2xs 0.86x, iq3xxs 0.83x of the reference. Open residue: the per-dword qs byte loads of iq2xs/iq3xxs. + 2026-09-01, ARM: the sdot lattice's decode composes row PAIRS straight from the grid words (two + rows x 8 weights per vector, one u64 grid entry per iq2 row, the ksigns formats signed by a +-1 + table row per 7-bit code, iq3s/iq2s by the mask off the plane's sign column) - M1 one thread: + iq2xs 1.24x, iq2xxs 1.20x, iq3xxs 1.36x, iq2s 1.51x, iq3s 1.15x, all five ahead. x86 is untouched + (zen2 v3: iq3s 1.36x, iq2xs 1.13x, iq2xxs 1.04x, iq2s 1.01x, iq3xxs 1.00x) - its residue stays + the qs byte loads; the M1 lesson (3 loads against 4 NEON ops per cycle, count both) is in the plan. 62. **IQ3_S Metal decode: the ~140 GB/s compose ceiling (tg 0.95x).** Eight GEMV forms raced at n=2048 d=8192 - gather placement x3, gather deleted, signs deleted, llama.cpp's exact @@ -825,3 +823,10 @@ through the sampler's state; (3) the single-token reconsideration penalty of entry 63. Nobody publishes what the effort levels cost; measure our three rungs on the 27B before naming them. +65. **An in-process image-off seam for facade-knob test cells.** A cell whose subject is a + facade lane knob (`load_asr_model` under `set_asr_tower_fp32`) must keep the facade + loader, and on a cold box that load BAKES the pinned lane's `.dlim` and GC-purges the + serving lane's flavor (`dlim_gc_stale`) - the class that broke `test_audio_embedder` + once already. The tests checklist ledgers the residue; the fix is an in-process + equivalent of `DASLLAMA_IMAGE=0` (`g_env_engine.image` is a `let` read at load), so such + cells can run image-free instead of risking the purge. diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index a876e25d2b..6fdd02fd55 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -243,6 +243,17 @@ the bound; the fix is the vectorized u64-lane compose (2 rows x 8 weights per NE 8 per 512-bit) with one pairwise add recovering the accumulator layout - one emitter change under the gather branch, no plane change, retires the panel and the parity chain. k6 0.90-0.91 on both x86 boxes stays open. +1. DONE ARM (2026-09-01, two commits): the row-pair decode under the sdot lattice + the ksigns +-1 + table rows + the whole-u64 grid load - M1 iq2xs 1.24x, iq2xxs 1.20x, iq3xxs 1.36x, iq2s 1.51x, + iq3s 1.15x (section 2, the LANDED notes). The x86 emission is byte-identical (gated DOT_SDOT). +2. NEXT x86, the row-QUAD twin for zen4's 0.77-0.88x: four rows x 8 weights per 256-bit vector + (four u64 grid loads, vmovq/vpinsrq/vinserti128), the activation's 8 bytes broadcast per lane + (vpbroadcastq) and SIGNED per row by `vpsignb` against the same +-1 table rows - maddubs needs + its unsigned operand, so on x86 the signs go on the activation copy, which the 4-rows-per-vector + layout allows where the 8-rows-per-vector panel layout did not. Then vpmaddubsw + the existing + fold. The zen2 `gather="reg"` that measured 1.85x slower was insertelement PER DWORD with GPR + sign math; this is four qword inserts and no sign arithmetic. Measure on zen2 first (all five + at 1.00-1.36x there - it must not lose), then zen4 (rent) where the gap lives. CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x. The memo (`kernel_parity_research_arm.md`): the sdot count is at parity (2 per 32 weights per row, both sides); From 6641a2ecbb446f716abf548dfe422f1bd277554f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 05:05:12 -0700 Subject: [PATCH 030/123] kernel parity, grid decode: one row-group emitter at any width - and the x86 insert form measured and killed emit_grid_rows_blk composes width/64 rows x 8 weights per weight vector, the +-1 sign bytes (the ksigns table row per code, or the plane's sign column as mask | 1) multiplied into the magnitudes; concat_rows and the activation broadcast follow the width. The same emitter with maddubs/vpdpbusd arms (signs on the activation copy through vpsignb) ran the x86 gemv on zen2: iq2xs 4646 -> 6785 us, iq2s 5056 -> 6826, iq3s 7612 -> 11576, iq3xxs 6589 -> 7995, iq2xxs 5144 -> 7142 - correct (TEST 90 ok) and 1.2-1.5x slower than the panel, and slower than llama.cpp's own AVX2 insert form (iq2xs 5320) in the same shape. The x86 arms are gone; grid_rows_path admits the sdot lattice only, where the M1 reads iq2xs 2633, iq2s 3250, iq3s 4938, iq3xxs 3689, iq2xxs 2911 (TEST 90 ok). The M1 ladder with the reference exe that carries the define: every one of the 32 rows at or above parity (decode floor k5 1.06x, tiles 1.92-15.2x) - the plan's section 2. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 122 +++++++++++------- modules/dasLLAMA/followup_general.md | 3 + plans/kernel_parity_pass.md | 37 +++++- 3 files changed, 111 insertions(+), 51 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 40915ab5a7..b9bc3c7b71 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -247,7 +247,7 @@ struct private TileEmit { psign_decl : LLVMOpaqueValue? // x64 width-256 sign apply (VPSIGNB) iq3s_panel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) - sign_code_pm1_g : LLVMOpaqueValue? // the sdot row-pair decode: ksigns code -> eight +-1 bytes, [128 x i64], parity bit folded in + sign_code_pm1_g : LLVMOpaqueValue? // the row-group grid decode: ksigns code -> eight +-1 bytes, [128 x i64], parity bit folded in gfn : LLVMOpaqueValue? // the function under emit (block creation for emitted inner loops) gctx : LLVMOpaqueContext? psign_ty : LLVMOpaqueType? @@ -909,8 +909,8 @@ def private load_f16_vec_at(var te : TileEmit; var base, off : LLVMOpaqueValue?; // the backend lowers the pair to a single addp) def private pairwise_add_i32(var te : TileEmit; var av, bv : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { let b = te.builder - let evens <- [0, 2, 4, 6] - let odds <- [1, 3, 5, 7] + let evens <- [for (i in range(te.rv)); 2 * i] + let odds <- [for (i in range(te.rv)); 2 * i + 1] var e = LLVMBuildShuffleVector(b, te.types, av, bv, evens, "") var o = LLVMBuildShuffleVector(b, te.types, av, bv, odds, "") return LLVMBuildAdd(b, e, o, name) @@ -1394,22 +1394,23 @@ def private sign_pm1_row(var te : TileEmit; var code : LLVMOpaqueValue?) : LLVMO return LLVMBuildBitCast(b, LLVMBuildLoad2Aligned(b, te.types.t_int64, gp, 8u, ""), LLVMVectorType(te.types.t_int8, 8u), "") } -//! the row pair's sign mask off the plane's sign column (byte r*4 + l per row): lanes 0-7 take row rA's -//! byte, 8-15 row rB's, bit k selects lane k - then (w ^ m) - m negates the masked weights -def private negate_pair(var te : TileEmit; var w, col : LLVMOpaqueValue?; rA, rB, l : int; name : string) : LLVMOpaqueValue? { +//! rows r0.. of a row group's +-1 sign bytes off the plane's sign column (iq3s/iq2s): byte r*4 + l +//! broadcast over the row's 8 lanes, bit k selects lane k, the mask | 1 +def private sign_col_pm1(var te : TileEmit; var col : LLVMOpaqueValue?; r0, l : int; name : string) : LLVMOpaqueValue? { let b = te.builder - let idx <- [for (i in range(16)); (i < 8 ? rA : rB) * 4 + l] + let w8 = te.width / 8 + let idx <- [for (i in range(w8)); (r0 + i / 8) * 4 + l] var bytes = LLVMBuildShuffleVector(b, te.types, col, col, idx, "sgp") - var bits : array - bits |> reserve(16) - for (i in range(16)) { - bits |> push(LLVMConstInt(te.types.t_int8, uint64(1 << (i % 8)), 0)) - } - var bitsv = LLVMConstVector(array_data_ptr(bits), 16u) - var m = LLVMBuildSExt(b, LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntEQ, LLVMBuildAnd(b, bytes, bitsv, ""), bitsv, ""), te.v16i8, "sgpm") - return LLVMBuildSub(b, LLVMBuildXor(b, w, m, ""), m, name) + var bits <- [for (i in range(w8)); LLVMConstInt(te.types.t_int8, uint64(1 << (i % 8)), 0)] + var bitsv = LLVMConstVector(array_data_ptr(bits), uint(w8)) + var m = LLVMBuildSExt(b, LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntEQ, LLVMBuildAnd(b, bytes, bitsv, ""), bitsv, ""), te.vwi8, "sgpm") + return LLVMBuildOr(b, m, splat_i8w(te, 1), name) } +//! the sdot lattice's grid decode composes row groups straight from the grid words; x86 keeps the panel (the +//! insert form measured 1.2-1.5x slower on zen2 - the ledger, followup 61) +def private grid_rows_path(te : TileEmit) : bool => te.dotKind == DOT_SDOT && te.rv == 4 + //! eight +-1 bytes for sign byte `sb`, lane k negative when bit k is set def private pm1_of(sb : int) : uint64 { var v = 0ul @@ -1422,7 +1423,7 @@ def private pm1_of(sb : int) : uint64 { //! the row-pair decode's sign table: the 128 ksigns codes with the code's parity as the 8th bit //! (ksigns[i] == i | parity(i) << 7), each as its eight +-1 bytes def private sign_pm1_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { - if (te.dotKind != DOT_SDOT) return // only the sdot lattice's row-pair decode reads it + if (!grid_rows_path(te)) return // only the row-group grid decode reads it te.sign_code_pm1_g = LLVMGetNamedGlobal(mod, "dasllama.sign.code_pm1") if (te.sign_code_pm1_g != null) return var codes : array @@ -1627,16 +1628,34 @@ def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { -//! the 8 activation bytes at byte offset `off` of token 0, replicated into both 64-bit lanes -//! (ld1r {v.2d} after lowering): the non-indexed sdot's operand for a [rowA 8w | rowB 8w] weight vector +//! the 8 activation bytes at byte offset `off` of token 0, replicated over every 64-bit lane of the +//! weight width (ld1r after lowering): the dot operand for a [row 8w | row 8w ...] weight vector def private bcast_qword_x(var te : TileEmit; var off : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { let b = te.builder + let n = te.width / 64 var p = LLVMBuildGEP2(b, te.types.t_int8, te.x[0], off, "") var q = LLVMBuildLoad2Aligned(b, te.types.t_int64, p, 1u, "") - var v2 = LLVMVectorType(te.types.t_int64, 2u) - var v = LLVMBuildInsertElement(b, LLVMGetUndef(v2), q, LLVMConstInt(te.types.t_int32, 0ul, 0), "") - var spl = LLVMBuildShuffleVector(b, te.types, v, v, [0, 0], "") - return LLVMBuildBitCast(b, spl, te.v16i8, name) + var vn = LLVMVectorType(te.types.t_int64, uint(n)) + var v = LLVMBuildInsertElement(b, LLVMGetUndef(vn), q, LLVMConstInt(te.types.t_int32, 0ul, 0), "") + let zeros <- [for (_i in range(n)); 0] + var spl = LLVMBuildShuffleVector(b, te.types, v, v, zeros, "") + return LLVMBuildBitCast(b, spl, te.vwi8, name) +} + +//! `rows` <8 x i8> vectors side by side as one weight-width vector (pairwise shuffles - ins after lowering) +def private concat_rows(var te : TileEmit; var rows : array; lanes : int = 8) : LLVMOpaqueValue? { + if (length(rows) == 1) return rows[0] + let b = te.builder + let idx <- [for (i in range(2 * lanes)); i] + var nxt <- [for (i in range(length(rows) / 2)); LLVMBuildShuffleVector(b, te.types, rows[2 * i], rows[2 * i + 1], idx, "")] + return concat_rows(te, nxt, 2 * lanes) +} + +//! one row group's sdot with its +-1 sign bytes multiplied into the magnitudes +def private grid_rows_dot(var te : TileEmit; var acc, w, sg, xv : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + var args <- [acc, LLVMBuildMul(b, w, sg, "ws"), xv] + return LLVMBuildCall2(b, te.sdot_ty, te.sdot_decl, args, "dot") } //! row `base`'s 8 raw grid weights for (blk, l) as <8 x i8>: the iq2 formats' u64 grid entry in one @@ -1653,40 +1672,43 @@ def private row_weights8(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l return LLVMBuildBitCast(b, v, v8i8, "") } -//! One block of a grid format's decode under the sdot lattice as row PAIRS: a <16 x i8> weight vector is two -//! rows x 8 weights (one u64 grid entry per row for iq2), signed by a mask off the plane's sign column (iq3s/iq2s) -//! or the ksigns +-1 table rows; the activation's 8 bytes replicated; one pairwise add per row quad yields the fold's [A, B, C, D]. -def private emit_grid_pairs_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var signCol : LLVMOpaqueValue? [2]; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { +//! One block of a grid format's decode as ROW GROUPS: a weight-width vector is width/64 rows x 8 weights +//! straight from the grid words (one u64 grid entry per iq2 row), signed by +-1 bytes - the ksigns table row +//! per code or the plane's sign column (iq3s/iq2s); one pairwise add per group pair yields the fold's per-row lanes. +def private emit_grid_rows_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var signCol : LLVMOpaqueValue? [2]; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { let b = te.builder - let concat <- [for (i in range(16)); i] + let n = te.width / 64 let colSigns = te.kq == 33 || te.kq == 23 for (qd in range(te.rq)) { - var acc : LLVMOpaqueValue? [2] // per row pair; the lo strip (l < 2) when iq2 + var acc : LLVMOpaqueValue? [2] // per row group; the lo strip (l < 2) when iq2 var acc1 : LLVMOpaqueValue? [2] // iq2: the hi strip (l >= 2) - for (pr in range(2)) { - acc[pr] = LLVMConstNull(te.vni32) - acc1[pr] = LLVMConstNull(te.vni32) + for (g in range(2)) { + acc[g] = LLVMConstNull(te.vni32) + acc1[g] = LLVMConstNull(te.vni32) } for (l in range(4)) { var xv = bcast_qword_x(te, LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32 + l * 8)), ""), "x8{blk}_{l}") - for (pr in range(2)) { - let rA = qd * te.rv + 2 * pr - var baseA = LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(rA * 4)), "") - var baseB = LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(rA * 4 + 4)), "") - var w = LLVMBuildShuffleVector(b, te.types, row_weights8(te, baseA, blk, l), row_weights8(te, baseB, blk, l), concat, "wp{blk}_{l}_{qd}_{pr}") - var ws : LLVMOpaqueValue? - if (colSigns) { - ws = negate_pair(te, w, signCol[qd], 2 * pr, 2 * pr + 1, l, "wps{blk}_{l}_{qd}_{pr}") - } else { - var sg = LLVMBuildShuffleVector(b, te.types, sign_pm1_row(te, kq_row_sign_code(te, baseA, blk, l)), sign_pm1_row(te, kq_row_sign_code(te, baseB, blk, l)), concat, "sp{blk}_{l}_{qd}_{pr}") - ws = LLVMBuildMul(b, w, sg, "wps{blk}_{l}_{qd}_{pr}") + for (g in range(2)) { + let r0 = g * n + var rows : array + var sgs : array + rows |> reserve(n) + sgs |> reserve(n) + for (i in range(n)) { + var base = LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64((qd * te.rv + r0 + i) * 4)), "") + rows |> push(row_weights8(te, base, blk, l)) + if (!colSigns) { + sgs |> push(sign_pm1_row(te, kq_row_sign_code(te, base, blk, l))) + } } - var args <- [(iq2 && l >= 2) ? acc1[pr] : acc[pr], ws, xv] - var d = LLVMBuildCall2(b, te.sdot_ty, te.sdot_decl, args, "sdotp") - if (iq2 && l >= 2) { - acc1[pr] = d + var w = concat_rows(te, rows) + var sg = colSigns ? sign_col_pm1(te, signCol[qd], r0, l, "sp{blk}_{l}_{qd}_{g}") : concat_rows(te, sgs) + let hi = iq2 && l >= 2 + var d = grid_rows_dot(te, hi ? acc1[g] : acc[g], w, sg, xv) + if (hi) { + acc1[g] = d } else { - acc[pr] = d + acc[g] = d } } } @@ -1709,8 +1731,8 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let iq2 = te.kq == 23 || te.kq == 24 // iq2s/iq2xs: per-16 UNSIGNED strips - lo/hi halves keep split accumulators let nl = te.kq == 45 // iq4nl: the 44 LUT decode with q40's PER-BLOCK f16 d fold (no sub-scales, no min term) let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first - // the sdot lattice's decode shape composes row PAIRS straight from the grid words (emit_grid_pairs_blk) - no panel - let pairs = gather && te.dotKind == DOT_SDOT && tokCount == 1 && te.rv == 4 + // the decode shape composes row GROUPS straight from the grid words (emit_grid_rows_blk) - no panel + let pairs = gather && tokCount == 1 && grid_rows_path(te) var wbase = te.wg var wb : LLVMOpaqueValue? var wbP : LLVMOpaqueValue? @@ -1767,7 +1789,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var } } if (pairs) { - emit_grid_pairs_blk(te, wbP, xb, blk, signCol, a, a1, iq2) + emit_grid_rows_blk(te, wbP, xb, blk, signCol, a, a1, iq2) } for (j in range(pairs ? 0 : 4)) { for (qd in range(rq)) { diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 338bb1f0a4..c69f5c7341 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -780,6 +780,9 @@ iq2xs 1.24x, iq2xxs 1.20x, iq3xxs 1.36x, iq2s 1.51x, iq3s 1.15x, all five ahead. x86 is untouched (zen2 v3: iq3s 1.36x, iq2xs 1.13x, iq2xxs 1.04x, iq2s 1.01x, iq3xxs 1.00x) - its residue stays the qs byte loads; the M1 lesson (3 loads against 4 NEON ops per cycle, count both) is in the plan. + 2026-09-01, x86: the same row-group form measured 1.2-1.5x SLOWER than the panel on zen2 (iq2xs + 4646 -> 6785 us; llama.cpp's AVX2 insert form does 5320 there) - killed for x86, kept for the sdot + lattice; zen4's 0.77-0.88x waits on a port profile (`plans/kernel_parity_pass.md`, queue). 62. **IQ3_S Metal decode: the ~140 GB/s compose ceiling (tg 0.95x).** Eight GEMV forms raced at n=2048 d=8192 - gather placement x3, gather deleted, signs deleted, llama.cpp's exact diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 6fdd02fd55..96096c4906 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -175,6 +175,32 @@ Decode below 1.0: k6 0.69, k5 0.95; at 1.0: k3 1.00, iq3xxs 1.00, iq2s 1.01. Eve M1 note: the ladder's reference must be `~/Work/llama.cpp/build/bin/test-backend-ops` (the fork checkout, 98c4764b6, carries the define); `build-cpu/` does not and ran every core - the ladder now refuses such a binary. +### M1 v3 - the full ladder after the ARM grid landing (2026-09-01, one thread, best of 5 interleaved rounds) + +`~/Work/llama.cpp/build/bin/test-backend-ops` (the fork checkout at 98c4764b6, the define in). EVERY row at or +above parity - the M1 CPU is closed for this pass. + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 1059 / 1562 | 1.48 | 214129 / 434961 | 2.03 | +| k4 | 839 / 1343 | 1.60 | 268955 / 703992 | 2.62 | +| k5 | 2067 / 2185 | 1.06 | 252434 / 1129322 | 4.47 | +| k6 | 1777 / 2401 | 1.35 | 401658 / 1243030 | 3.09 | +| q40 | 863 / 1612 | 1.87 | 276856 / 531679 | 1.92 | +| q51 | 2075 / 2749 | 1.33 | 453329 / 1436227 | 3.17 | +| iq4xs | 988 / 1680 | 1.70 | 262229 / 870313 | 3.32 | +| k3 | 1925 / 2619 | 1.36 | 486461 / 1348956 | 2.77 | +| iq3s | 4914 / 5680 | 1.16 | 192154 / 2922617 | 15.21 | +| iq3xxs | 3572 / 4902 | 1.37 | 192186 / 2492096 | 12.97 | +| iq4nl | 1008 / 1847 | 1.83 | 290154 / 949701 | 3.27 | +| k2 | 1348 / 1915 | 1.42 | 357321 / 986778 | 2.76 | +| iq2s | 3259 / 4915 | 1.51 | 255732 / 2516761 | 9.84 | +| iq2xs | 2635 / 3249 | 1.23 | 255582 / 1669316 | 6.53 | +| iq2xxs | 2871 / 3435 | 1.20 | 192075 / 1765362 | 9.19 | +| mx4 | 1271 / 1900 | 1.50 | 331361 / 978109 | 2.95 | + +Decode floor k5 1.06x; the grid formats 1.16-1.51x (from 0.51-0.93x before the row-pair decode). Tiles 1.92-15.2x. + ## 3. Research memos (read before touching the kernels) - `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, @@ -246,7 +272,14 @@ stays open. 1. DONE ARM (2026-09-01, two commits): the row-pair decode under the sdot lattice + the ksigns +-1 table rows + the whole-u64 grid load - M1 iq2xs 1.24x, iq2xxs 1.20x, iq3xxs 1.36x, iq2s 1.51x, iq3s 1.15x (section 2, the LANDED notes). The x86 emission is byte-identical (gated DOT_SDOT). -2. NEXT x86, the row-QUAD twin for zen4's 0.77-0.88x: four rows x 8 weights per 256-bit vector +2. DONE-KILLED x86 (2026-09-01): the row-QUAD form measured on zen2, us, panel -> rows: iq2xs 4646 -> + 6785, iq2s 5056 -> 6826, iq3s 7612 -> 11576, iq3xxs 6589 -> 7995, iq2xxs 5144 -> 7142 (TEST 90 ok, the + form is correct). llama.cpp's own AVX2 grid path IS this insert form and runs iq2xs at 5320 there - so + the insert shape loses to the panel on zen2 AND ours is slower than theirs in the same shape (the sign + table's four scalar loads + inserts where theirs shuffles; the per-row index math). It cannot close + zen4's 0.77-0.88x by being the reference's shape; the emitter keeps it for the sdot lattice only + (`grid_rows_path`). zen4 needs a profile first (which port saturates) - item 3. + The plan that was: four rows x 8 weights per 256-bit vector (four u64 grid loads, vmovq/vpinsrq/vinserti128), the activation's 8 bytes broadcast per lane (vpbroadcastq) and SIGNED per row by `vpsignb` against the same +-1 table rows - maddubs needs its unsigned operand, so on x86 the signs go on the activation copy, which the 4-rows-per-vector @@ -254,6 +287,8 @@ stays open. fold. The zen2 `gather="reg"` that measured 1.85x slower was insertelement PER DWORD with GPR sign math; this is four qword inserts and no sign arithmetic. Measure on zen2 first (all five at 1.00-1.36x there - it must not lose), then zen4 (rent) where the gap lives. +3. NEXT x86 zen4 grid 0.77-0.88x: rent the c7a again with `perf stat` (uops per port, store-forwarding + stalls) on iq2xs/iq3xxs decode against the reference exe's same shape - one proven fact before any form. CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x. The memo (`kernel_parity_research_arm.md`): the sdot count is at parity (2 per 32 weights per row, both sides); From 0f07299643c8c566112c6ce9322535790d4182fa Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 05:26:45 -0700 Subject: [PATCH 031/123] kq_kernel_bench: --base-align and --base-offset put the arena at a chosen page phase - k6's per-process swing is the phase k6 decode on zen2 swings 4.0-6.3 ms from one fresh process to the next while k4, k5, k3 and the reference q6_K hold within 4%; pinned to one core it still swings, so it follows the process. With the arena base page-aligned (the engine's image planes start on 16 KiB boundaries) it reads 4862-6364 us eight times out of eight, where the heap's 64-byte draws run 4.0-4.3 two times in three; the sub-page offset sweep (0..3840 by 256) reads 4383-6305 with no band. The JIT'd k6 body carries 72 ymm spill stores and 95 stack reloads per superblock (k5 20 + 8, k3 42 + 13); pinning its qh column loads cut the stores to 27 and made every phase slow, so it stays out. The flags make the engine's phase the bench's default question; the plan carries the numbers and the next step (counters). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 3 +++ .../benchmarks/matmul/kq_kernel_bench.das | 16 ++++++++++++---- plans/kernel_parity_pass.md | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index 48dae6595b..a5f5ae2514 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -269,6 +269,9 @@ per try), and the reference row at the same shape is the reference exe's `test-b `harness/kernel_ladder.sh` runs both sides for every format and prints the box's ratio table. The app run comes only after a spelling wins there. Procedure, fact base and work queue: `plans/kernel_parity_pass.md`. +2026-09-01: the pass closed the M1 CPU at kernel level - all 32 ladder rows at or above the +reference (the grid decodes 1.16-1.51x, from 0.51-0.93x) - so the per-format M1 CPU tg tails stamped +below (0.51x-0.73x) predate the ARM row-group decode; re-stamp the vehicles before quoting them. A real file whose every tensor type is now loadable (the header census script in the session scratchpad, or `harness/gguf_dump.das`), through `examples/dasLLAMA/run.das` against diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index fccc5ad4e3..809c205f89 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -45,6 +45,10 @@ struct BenchArgs { perm : string @clarg_doc = "TSV rows only, no banner" tsv : bool + @clarg_doc = "Arena base alignment in bytes (64; 4096 fixes the planes' phase against page boundaries, which the heap otherwise draws per process)" + base_align : int = 64 + @clarg_doc = "Bytes added past the aligned arena base (with --base-align 4096: the planes' phase inside the page)" + base_offset : int = 0 } typedef GemvFn = function<(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void> @@ -155,6 +159,8 @@ def plane_shape(f : KqFmt; mx4 : bool; n : int64) : PlaneShape { return PlaneShape(wq_row = nsb * kq_qsb(f), tq_row = nsb * tile_kq_bytes_per_sb(f), ws_row = nsb * kq_ssb(f)) } +var BASE_ALIGN = 64l // the arena's start alignment - --base-align; the planes' phase against 4 KB pages follows it +var BASE_OFFSET = 0l // --base-offset, added past the aligned start let ALIGN = 64l // the image's planes are page-aligned (ARCHITECTURE_IMAGE.md, image-page-alignment); a 64-byte load split across cache lines is what Intel penalizes 2x let STAGGER = 1088l // plane starts differ in their low 12 address bits (1088 = 1024 + 64 keeps six starts distinct mod 4096): planes that alias in the L1/L2 set logic make a run's time depend on where the heap put them let SCALE_BYTE = uint8(0x3C) // as f16 0x3C3C = 1.06, as f32 0x3C3C3C3C = 0.0115, as e8m0 2^-67, as a 6-bit sub-scale 60 - normal in every scale form @@ -179,10 +185,10 @@ def next_plane(var end : int64&; bytes : int64; slot : int) : int64 { return start } -//! the arena's first 64-byte-aligned byte +//! the arena's first BASE_ALIGN-aligned byte def arena_base(var p : Planes) : int64 { unsafe { - return (ALIGN - int64(intptr(addr(p.arena[0])) & uint64(ALIGN - 1l))) & (ALIGN - 1l) + return ((BASE_ALIGN - int64(intptr(addr(p.arena[0])) & uint64(BASE_ALIGN - 1l))) & (BASE_ALIGN - 1l)) + BASE_OFFSET } } @@ -195,8 +201,8 @@ def fill_planes(var p : Planes; sh : PlaneShape; n, d, ntok : int64) { p.xs = next_plane(end, ntok * n / 32l * 4l, 4) p.xbs = next_plane(end, ntok * n / 16l * 4l, 5) p.y = next_plane(end, ntok * d * 4l, 6) - p.arena |> reserve(end + ALIGN) - p.arena |> resize(end + ALIGN) + p.arena |> reserve(end + BASE_ALIGN + BASE_OFFSET) + p.arena |> resize(end + BASE_ALIGN + BASE_OFFSET) let base = arena_base(p) p.kq += base p.tkq += base @@ -534,6 +540,8 @@ def formats_of(spec : string) : array> { def main : int { var cfg = BenchArgs() let rc = parse_args_with_help(cfg, "kq_kernel_bench") + BASE_ALIGN = int64(cfg.base_align) + BASE_OFFSET = int64(cfg.base_offset) return rc if (rc >= 0) if (!jit_enabled()) { to_log(LOG_ERROR, "run with -jit: the kq generators only fire under the JIT\n") diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 96096c4906..e3d4995f45 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -240,6 +240,24 @@ k6 0.90x, k5 0.91x): 1109 ms) so it is decode-only; the principled form is the memo's D1 group-major reorder. 2. OPEN k6 0.87-0.90x, k5 0.87-0.91x: the pin gained nothing on k6; the memo (research_cpu_kquant.md) sees no clean lever for k5 (D4) and offers a sub-block unroll knob (D5) for both. + 2026-09-01, the k6 swing dissected (zen2, one thread, fresh process each, us): alone 4153 / 4225 / + 5468 / 4711; pinned to ONE core (start /affinity) 4040 / 5804 / 4220 / 4252 and 4138 / 4368 / 5632 / + 5200 - it follows the process, not the core. k4 (1999-2005), k5 (3590-3760), k3 (2648-2850) and the + reference q6_K (3664-3768) do not swing. Arena base page-aligned (`--base-align 4096`): 4862 / 5971 + / 6052 / 5916, at 65536: 6024 / 6364 / 5607 / 5567 - eight of eight slow, where the heap's 64-byte + draws run 4.0-4.3 two times in three. The sub-page phase sweep (`--base-offset` 0..3840 by 256, one + process each) reads 4383-6305 with no band (3840 -> 4383, 3584 -> 6231). Reading: the per-process + draw is the L1-set phase of k6's spill slots (the native stack, ASLR) against the planes; k6 has the + largest live set of the four (memo: 2 qh + 4 ql + 8 chains). The engine's image planes start on 16 KiB + boundaries (ARCHITECTURE_IMAGE.md 2.1a), i.e. the slow phase - the engine's k6 decode is ~0.61x, not + the ladder's 0.69-0.87. Spills counted in the cached JIT DLL (`objdump -d` over the `.map` range of + the `implementation` symbol): k6 72 ymm spill stores + 95 stack ymm reloads per superblock body (1224 + instructions, 64 maddubs) against k5 20 + 8 and k3 42 + 13. Pinning the qh column loads (volatile, the + k3 cure) cut the stores to 27 but left 90 reloads and made k6 uniformly SLOW - engine phase 5537-5895, + heap phase 5574-6078 (the 4.0-4.3 mode gone), tile 634 -> 669 ms - so it was reverted; the memo's + "the pin gained nothing on k6" stands, and the fast mode is not the spill count. NEXT (its own session, + with counters - AMD uProf on this box): what the 4.0 and 6.0 modes differ in, at `--base-align 4096` + with `--base-offset` as the phase knob; only then D2's hoist / D5's unroll. 3. Noise: the one-thread bench and the reference both drift ~5-8% run to run; interleaved rounds hold ours steady, the reference is re-run per table. iq3xxs/iq2s/iq2xxs tie at 1.00; k2 now clear. From 50650f2587cfbecb880b4cfa7d25f5e2767315a1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 06:57:40 -0700 Subject: [PATCH 032/123] kq_kernel_bench --team N: the GEMV dispatched the engine's way - N row chunks self-served by the jobque team, wall = the slowest lane, the lanes' fastest and slowest call in two more columns The one-thread raw call measures the kernel; the engine runs a decode GEMV as team_parallel_for over row chunks on the worker threads and waits for the slowest lane. --team N (with DAS_JOBQUE_THREADS=N; kernel_ladder.sh TEAM=N puts the reference at GGML_BENCH_THREADS=N) times that shape: chunk edges on 32-row units, per-lane call time by slot into a pre-sized array (no allocation on a lane), lane_lo / lane_hi appended to the TSV row. zen2 16 lanes, engine phase: k6 wall 631-918 us against the reference's 442 with lanes 134 -> 1321 us; k4 258-374 against 248 with lanes 84 -> 891. The one-thread ladder's k4 1.21x is 0.66-0.96x at this shape - the slowest lane, not the kernel, sets the wall. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../benchmarks/matmul/kq_kernel_bench.das | 84 ++++++++++++++++--- modules/dasLLAMA/harness/kernel_ladder.sh | 11 ++- 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 809c205f89..6cd56b7bac 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -6,6 +6,7 @@ options _dasllama_internal = true require daslib/clargs require daslib/enum_trait require daslib/fio +require daslib/jobque_boost require dasllama/dasllama_math_gen require dasllama/dasllama_math require dasllama/dasllama_kqformat @@ -49,6 +50,8 @@ struct BenchArgs { base_align : int = 64 @clarg_doc = "Bytes added past the aligned arena base (with --base-align 4096: the planes' phase inside the page)" base_offset : int = 0 + @clarg_doc = "GEMV the engine's way: N row chunks self-served by the jobque team (DAS_JOBQUE_THREADS lanes), wall = the slowest lane, the lanes' fastest and slowest call in two extra columns; 0 = one raw call on this thread" + team : int = 0 } typedef GemvFn = function<(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void> @@ -243,6 +246,22 @@ def fill_planes(var p : Planes; sh : PlaneShape; n, d, ntok : int64) { struct RowStat { perm : string us : array + lane_lo : int64 // --team: the fastest single lane call seen across the rounds + lane_hi : int64 // --team: the slowest +} + +let TEAM_UNIT = 32l // --team chunk edges land on 32-row boundaries - every registry mr divides them + +//! the engine's dispatch shape: `chunks` row ranges self-served by the team's lanes, each lane adding its call time to laneP[slot] +def team_gemv(fn : GemvFn; units, chunks : int; var yp : float?; kqp, ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64; var laneP : int64?) { + team_parallel_for_indexed(0, units, chunks) <| @(slot : int; jb : int; je : int) { + let l0 = ref_time_ticks() + var y = yp + invoke(fn, y, kqp, ksp, xqp, xsp, xbsp, n, int64(jb) * TEAM_UNIT, int64(je) * TEAM_UNIT) + unsafe { + laneP[slot] += int64(get_time_usec(l0)) + } + } } def median_us(var us : array) : int64 { @@ -265,7 +284,8 @@ def emit_row(var rows : int&; name, tier : string; var st : RowStat; n, d, ntok, let gflops = 2.0 * float(n) * float(d) * float(ntok) / (float(best) * 1000.0) let mode = get_env_variable("DAS_TUNE_MODE") let perm = (st.perm == "reference" && mode != "tune" && mode != "test") ? "stamped" : st.perm // normal mode: the registry's one row IS the stamped winner - print("{name}\t{tier}\t{perm}\t{n}\t{d}\t{ntok}\t{best}\t{med}\t{ns_per_sb}\t{gbs}\t{gflops}\n") + let lanes = st.lane_hi > 0l ? "\t{st.lane_lo}\t{st.lane_hi}" : "" + print("{name}\t{tier}\t{perm}\t{n}\t{d}\t{ntok}\t{best}\t{med}\t{ns_per_sb}\t{gbs}\t{gflops}{lanes}\n") rows++ } @@ -320,8 +340,36 @@ def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va let xsp = addr(p.arena[p.xs]) let xbsp = addr(p.arena[p.xbs]) var yp = addr(p.arena[p.y]) - time_rows(stats, cfg.rounds) $(row : int) { - invoke(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) + if (cfg.team > 0) { + let units = int(d / TEAM_UNIT) + var laneUs : array + laneUs |> resize(get_total_hw_jobs() + 1) + var laneP = addr(laneUs[0]) + for (st, fn in stats, fns) { + st.us |> reserve(cfg.rounds) + st.lane_lo = 999999999l + team_gemv(fn, units, cfg.team, yp, kqp, ksp, xqp, xsp, xbsp, n, laneP) // warmup + codegen, unmeasured + } + for (_r in range(cfg.rounds)) { + for (st, fn in stats, fns) { + for (u in laneUs) { + u = 0l + } + let t0 = ref_time_ticks() + team_gemv(fn, units, cfg.team, yp, kqp, ksp, xqp, xsp, xbsp, n, laneP) + st.us |> push(int64(get_time_usec(t0))) + for (u in laneUs) { + if (u > 0l) { + st.lane_lo = min(st.lane_lo, u) + st.lane_hi = max(st.lane_hi, u) + } + } + } + } + } else { + time_rows(stats, cfg.rounds) $(row : int) { + invoke(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) + } } } for (st in stats) { @@ -553,9 +601,28 @@ def main : int { } var fs <- formats_of(cfg.fmt) if (!cfg.tsv) { - print("kq_kernel_bench: n={cfg.n} d={cfg.d} ntok={cfg.ntok} rounds={cfg.rounds} tune_mode={get_env_variable("DAS_TUNE_MODE")}\n") - print("fmt\ttier\tperm\tn\td\tntok\tbest_us\tmedian_us\tns_per_sb\tGB/s\tGFLOP/s\n") + print("kq_kernel_bench: n={cfg.n} d={cfg.d} ntok={cfg.ntok} rounds={cfg.rounds} team={cfg.team} tune_mode={get_env_variable("DAS_TUNE_MODE")}\n") + print("fmt\ttier\tperm\tn\td\tntok\tbest_us\tmedian_us\tns_per_sb\tGB/s\tGFLOP/s{cfg.team > 0 ? "\tlane_lo\tlane_hi" : ""}\n") } + var rows = 0 + if (cfg.team > 0) { + with_job_que() { + setup_dasllama_jobque_() + rows = run_formats(cfg, fs) + } + } else { + rows = run_formats(cfg, fs) + } + delete fs + if (rows == 0) { + to_log(LOG_ERROR, "kq_kernel_bench: no rows - nothing matched --fmt '{cfg.fmt}' / --perm '{cfg.perm}' or every arm was skipped\n") + return 1 + } + return 0 +} + +//! every selected format's arms; the row count printed +def run_formats(cfg : BenchArgs; fs : array>) : int { var rows = 0 let nsb = int64(cfg.n) / 256l for (fe in fs) { @@ -574,10 +641,5 @@ def main : int { } delete p } - delete fs - if (rows == 0) { - to_log(LOG_ERROR, "kq_kernel_bench: no rows - nothing matched --fmt '{cfg.fmt}' / --perm '{cfg.perm}' or every arm was skipped\n") - return 1 - } - return 0 + return rows } diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh index a8a89e44b7..a1606ab941 100644 --- a/modules/dasLLAMA/harness/kernel_ladder.sh +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -11,6 +11,8 @@ # LCPP_TBO test-backend-ops (default: $HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe) # NTOK prefill tokens (default 512; 0 = decode rows only) # ROUNDS interleaved rounds per row (default 5) +# TEAM lanes: the bench dispatches the GEMV the engine's way (--team N, DAS_JOBQUE_THREADS=N) +# and the reference runs GGML_BENCH_THREADS=N; unset = one thread, one raw call # Output: a TSV table on stdout - fmt tier perm ours_us ours_med_us ref_us ratio - plus the box line. set -e -o pipefail # a failing daslang must not hide behind the join ROOT=$(cd "$(dirname "$0")/../../.." && pwd) # the tree this script lives in, whatever the env says @@ -22,6 +24,9 @@ TBO=${LCPP_TBO:-$HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-op FMTS=${1:-all} NTOK=${NTOK:-512} ROUNDS=${ROUNDS:-5} +TEAM=${TEAM:-0} +THREADS=$(( TEAM > 0 ? TEAM : 1 )) +TEAM_ARGS=""; [ "$TEAM" -gt 0 ] && TEAM_ARGS="--team $TEAM" WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT @@ -41,8 +46,8 @@ echo "# kernel ladder $(date +%F) box=$(hostname) daslang=$DASLANG ref=$TBO [ -x "$TBO" ] || { echo "kernel_ladder: no test-backend-ops at '$TBO' (set LCPP_TBO)" >&2; exit 1; } grep -q GGML_BENCH_THREADS "$TBO" || { echo "kernel_ladder: '$TBO' lacks the GGML_BENCH_THREADS define - it would run every core and the ratio column would lie (harness README)" >&2; exit 1; } set +e -DAS_TUNE_MODE=normal DAS_JOBQUE_THREADS=1 "$DASLANG" -jit "$ROOT/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das" \ - -- --fmt "$FMTS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv > "$WORK/ours.raw" 2> "$WORK/ours.err" +DAS_TUNE_MODE=normal DAS_JOBQUE_THREADS=$THREADS "$DASLANG" -jit "$ROOT/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das" \ + -- --fmt "$FMTS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv $TEAM_ARGS > "$WORK/ours.raw" 2> "$WORK/ours.err" rc=$? set -e awk -F'\t' 'NF >= 11' "$WORK/ours.raw" > "$WORK/ours.tsv" || true # the TSV rows; engine and tune notes fall away @@ -57,7 +62,7 @@ fi for f in $(cut -f1 "$WORK/ours.tsv" | sort -u); do t=$(ggml_type "$f") [ -n "$t" ] || { echo "kernel_ladder: no ggml type for '$f'" >&2; continue; } - GGML_BENCH_THREADS=1 "$TBO" perf -b CPU -o MUL_MAT -p "type_a=$t,type_b=f32,m=4096,n=(1|512)," 2>/dev/null \ + GGML_BENCH_THREADS=$THREADS "$TBO" perf -b CPU -o MUL_MAT -p "type_a=$t,type_b=f32,m=4096,n=(1|512)," 2>/dev/null \ | sed 's/\x1b\[[0-9;]*m//g' | awk -v f="$f" ' /MUL_MAT\(/ { n = $0; sub(/.*,n=/, "", n); sub(/,.*/, "", n) From 4af2d6e58e0a0dd820e06ff0a3a8f9aa0aeb415b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 07:21:14 -0700 Subject: [PATCH 033/123] k6 planes, CPU flavor: the qh half packed per sub-block - one byte carries a sub-block's four j sites at 2j The .dlim is baked for the hardware it runs on (Vulkan and Metal bake their own; the flavors never co-exist), so the CPU flavor's k6 plane is the CPU kernels' to lay out. Its qh half was GGUF's packing copied verbatim - four SUB-BLOCKS per byte - which is why the gemv kept a group's eight qh columns live across four sub-blocks (72 ymm spill stores + 95 stack reloads per superblock) or reloaded them (the volatile pin, measured worse). Column 2blk + half now holds that sub-block's four j sites at bit 2j: the same 64 bytes, two qh loads per sub-block, a uniform shift, nothing living past its sub-block. Six readers move together: repack_k6_grp, unpack_kq_panel_grp (the tile's byte-expand), the gemv arm, dequant_kq_row_grp, the generic tail dot, and the layout's grp -> disk inverse; IMAGE_VERSION 27 re-bakes CPU images, LLVM_JIT_CODEGEN_VERSION 0x5b re-keys the caches. gen_tune_probe TEST 90/90; test_gguf_quant, test_repack, test_kquant green. zen2 k6 decode, us: engine phase (page-aligned planes) 4444 / 4619 / 4708 from 4862-6364, heap phase 4749 / 4729 / 5427 from 4040-5804; the body 1161 instructions with 36 + 67 spill ops from 1224 with 72 + 95; TEAM=16 wall and the tile unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 16 ++++++------- modules/dasLLAMA/dasllama/dasllama_image.das | 2 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 11 +++++++-- .../dasllama/dasllama_math_default.das | 12 +++++----- .../dasLLAMA/dasllama/dasllama_math_gen.das | 12 +++++----- modules/dasLLAMA/dasllama/dasllama_repack.das | 23 +++++++++++-------- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- plans/kernel_parity_pass.md | 13 +++++++++++ 8 files changed, 58 insertions(+), 33 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index b9bc3c7b71..e78f09e556 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1082,14 +1082,14 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f wlo = or_bit_x10(te, wlo, hb, maskLo, "wlo5{blk}_{j * rq + qd}") whi = or_bit_x10(te, whi, hb, maskHi, "whi5{blk}_{j * rq + qd}") } elif (k6 && !te.kqBytes) { - let g2 = (blk % 4) * 2 - let hbase = 128 * mr + ((blk / 4) * 8) * 4 * mr - var hloOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + j * 4 * mr + qd * w8)), "") - var hhiOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + (j + 4) * 4 * mr + qd * w8)), "") - var qhlo = load_vec(te, te.vwi8, te.wg, hloOff, "qhl{blk / 4}_{j * rq + qd}") - var qhhi = load_vec(te, te.vwi8, te.wg, hhiOff, "qhh{blk / 4}_{j * rq + qd}") - var dlo = LLVMBuildAnd(b, LLVMBuildLShr(b, qhlo, splat_i8w(te, g2), ""), splat_i8w(te, 3), "") - var dhi = LLVMBuildAnd(b, LLVMBuildLShr(b, qhhi, splat_i8w(te, g2), ""), splat_i8w(te, 3), "") + // the plane's qh columns 2blk / 2blk + 1 hold this sub-block's four j sites at 2j: two loads per sub-block, nothing lives past it + let hbase = 128 * mr + blk * 2 * 4 * mr + var hloOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + qd * w8)), "") + var hhiOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + 4 * mr + qd * w8)), "") + var qhlo = load_vec(te, te.vwi8, te.wg, hloOff, "qhl{blk}_{qd}") + var qhhi = load_vec(te, te.vwi8, te.wg, hhiOff, "qhh{blk}_{qd}") + var dlo = LLVMBuildAnd(b, LLVMBuildLShr(b, qhlo, splat_i8w(te, 2 * j), ""), splat_i8w(te, 3), "") + var dhi = LLVMBuildAnd(b, LLVMBuildLShr(b, qhhi, splat_i8w(te, 2 * j), ""), splat_i8w(te, 3), "") wlo = LLVMBuildOr(b, wlo, LLVMBuildShl(b, dlo, splat_i8w(te, 4), ""), "wlo6{blk}_{j * rq + qd}") whi = LLVMBuildOr(b, whi, LLVMBuildShl(b, dhi, splat_i8w(te, 4), ""), "whi6{blk}_{j * rq + qd}") } diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index 88d36e48b9..a595c38c81 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 26 // the iq2xxs plane pair (IQ2_XXS native tier) +let IMAGE_VERSION = 27 // k6's qh plane packed per sub-block (CPU flavor) //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 920696275b..4e2dd9cfbd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -1000,9 +1000,16 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (bj in range64(32l)) { dq[128l + bj] = gq[128l * mr + bj * mr + r] } - } elif (fmt == KqFmt.k6) { + } elif (fmt == KqFmt.k6) { // qh columns 2blk + half carry one sub-block's four j sites at 2j; the disk byte h*32 + half*16 + j*4 + t carries sub-blocks 4h.. at 2b for (p in range64(64l)) { - dq[128l + p] = gq[128l * mr + ((p / 4l) * mr + r) * 4l + p % 4l] + let half = (p % 32l) / 16l + let j = (p % 16l) / 4l + var byte = 0u + for (bb in range64(4l)) { + let col = ((p / 32l) * 4l + bb) * 2l + half + byte |= ((uint(gq[128l * mr + (col * mr + r) * 4l + p % 4l]) >> uint(2l * j)) & 3u) << uint(2l * bb) + } + dq[128l + p] = uint8(byte) } } if (fmt == KqFmt.k6 || fmt == KqFmt.k3) { diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 126d1f932b..8165697848 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -1360,7 +1360,7 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m } for (j in range64(4l)) { let hb = fmt == 5l ? int(kqg[qb + 128l * mr + (blk * 4l + j) * mr + r]) : 0 - let g2 = uint((blk % 4l) * 2l) + let sj = uint(2l * j) for (t in range64(4l)) { let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) var qlo = int(nib & 15u) @@ -1369,11 +1369,11 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m qlo |= ((hb >> int(t)) & 1) << 4 qhi |= ((hb >> int(4l + t)) & 1) << 4 } elif (fmt == 6l) { - let hbase = qb + 128l * mr + (blk / 4l) * 32l * mr - let h0 = uint(kqg[hbase + (j * mr + r) * 4l + t]) - let h1 = uint(kqg[hbase + ((j + 4l) * mr + r) * 4l + t]) - qlo |= int(((h0 >> g2) & 3u) << 4u) - qhi |= int(((h1 >> g2) & 3u) << 4u) + let hbase = qb + 128l * mr + blk * 2l * mr * 4l // qh columns 2blk / 2blk + 1 hold this sub-block's four j sites at 2j + let h0 = uint(kqg[hbase + r * 4l + t]) + let h1 = uint(kqg[hbase + (mr + r) * 4l + t]) + qlo |= int(((h0 >> sj) & 3u) << 4u) + qhi |= int(((h1 >> sj) & 3u) << 4u) } if (fmt == 6l) { dst[kb + j * 4l + t] = s0 * (float(qlo) - 32.0) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 27f4e2932d..ef5b799629 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -179,7 +179,7 @@ def kq_grp_row_dot(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : var ihi = 0 for (j in range64(4l)) { let hb = fmt == 5l ? int(kqg[qb + 128l * mr + (blk * 4l + j) * mr + r]) : 0 - let g2 = uint((blk % 4l) * 2l) + let sj = uint(2l * j) for (t in range64(4l)) { let nib = uint(kqg[qb + ((blk * 4l + j) * mr + r) * 4l + t]) var qlo = int(nib & 15u) @@ -188,11 +188,11 @@ def kq_grp_row_dot(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : qlo |= ((hb >> int(t)) & 1) << 4 qhi |= ((hb >> int(4l + t)) & 1) << 4 } elif (fmt == 6l) { - let hbase = qb + 128l * mr + (blk / 4l) * 32l * mr - let h0 = uint(kqg[hbase + (j * mr + r) * 4l + t]) - let h1 = uint(kqg[hbase + ((j + 4l) * mr + r) * 4l + t]) - qlo |= int(((h0 >> g2) & 3u) << 4u) - qhi |= int(((h1 >> g2) & 3u) << 4u) + let hbase = qb + 128l * mr + blk * 2l * mr * 4l // qh columns 2blk / 2blk + 1 hold this sub-block's four j sites at 2j + let h0 = uint(kqg[hbase + r * 4l + t]) + let h1 = uint(kqg[hbase + (mr + r) * 4l + t]) + qlo |= int(((h0 >> sj) & 3u) << 4u) + qhi |= int(((h1 >> sj) & 3u) << 4u) } ilo += qlo * int(xqp[b * 32l + j * 4l + t]) ihi += qhi * int(xqp[b * 32l + 16l + j * 4l + t]) diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index b44d1470dd..14675c2ddb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -314,10 +314,15 @@ def repack_k6_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { } } } - for (h in range64(2l)) { // qh bytes verbatim, row-interleaved by 4-byte columns - for (jj in range64(8l)) { + for (blk in range64(8l)) { // qh re-packed per sub-block: column 2blk + half holds the four j sites' 2-bit fields at 2j + for (half in range64(2l)) { for (t in range64(4l)) { - kq[dq + 128l * mr + ((h * 8l + jj) * mr + r) * 4l + t] = tqp[sq + 128l + h * 32l + jj * 4l + t] + var byte = 0u + for (j in range64(4l)) { + let l = (blk / 4l) * 32l + half * 16l + j * 4l + t + byte |= ((uint(tqp[sq + 128l + l]) >> uint(2l * (blk % 4l))) & 3u) << uint(2l * j) + } + kq[dq + 128l * mr + ((blk * 2l + half) * mr + r) * 4l + t] = uint8(byte) } } } @@ -981,9 +986,9 @@ def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, n let qb = sbi * (fmt == 5l ? 160l : 192l) * mr let ob = sbi * 256l * mr for (blk in range64(8l)) { - let g2 = uint64((blk % 4l) * 2l) - let hbase = qb + 128l * mr + (blk / 4l) * 32l * mr // k6 high plane (k5 ignores) + let hbase = qb + 128l * mr + blk * 2l * mr * 4l // k6 high plane: columns 2blk (lo half) and 2blk + 1 (hi); k5 ignores for (j in range64(4l)) { + let sj = uint64(2l * j) let gcol = ((blk * 4l + j) * mr) * 4l var r4 = 0l while (r4 < mr) { @@ -1007,12 +1012,12 @@ def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, n hiw[w] = ((n >> 4ul) & LO4) | dh } } else { - let h0w = reinterpret(kqg + (hbase + (j * mr + r4) * 4l)) - let h1w = reinterpret(kqg + (hbase + ((j + 4l) * mr + r4) * 4l)) + let h0w = reinterpret(kqg + (hbase + r4 * 4l)) + let h1w = reinterpret(kqg + (hbase + (mr + r4) * 4l)) for (w in range64(2l)) { let n = nw[w] - low[w] = (n & LO4) | (((h0w[w] >> g2) & TOP2) << 4ul) - hiw[w] = ((n >> 4ul) & LO4) | (((h1w[w] >> g2) & TOP2) << 4ul) + low[w] = (n & LO4) | (((h0w[w] >> sj) & TOP2) << 4ul) + hiw[w] = ((n >> 4ul) & LO4) | (((h1w[w] >> sj) & TOP2) << 4ul) } } r4 += 4l diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 721baab44a..69e52799bd 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5aul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5bul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index e3d4995f45..02e6dabf68 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -258,6 +258,19 @@ k6 0.90x, k5 0.91x): "the pin gained nothing on k6" stands, and the fast mode is not the spill count. NEXT (its own session, with counters - AMD uProf on this box): what the 4.0 and 6.0 modes differ in, at `--base-align 4096` with `--base-offset` as the phase knob; only then D2's hoist / D5's unroll. + 2026-09-01, Boris's ruling - the .dlim is hardware-specific (Vulkan and Metal bake their own; the + flavors never co-exist), so the CPU flavor's planes are the CPU kernels' to lay out. k6's qh half + was GGUF's packing copied verbatim (four SUB-BLOCKS per byte), which is why the kernel kept eight qh + columns live across four sub-blocks or reloaded them. LANDED: the qh plane re-packed per sub-block + (column 2blk + half holds that sub-block's four j sites at 2j) - same 64 bytes, two qh loads per + sub-block, uniform shift, nothing lives past its sub-block; six readers moved (repack, tile unpack, + gemv arm, row dequant, generic tail dot, layout inverse), IMAGE_VERSION 27, JIT 0x5b. TEST 90/90, + test_gguf_quant / test_repack / test_kquant green. zen2 k6 decode, us: engine phase 4444 / 4619 / + 4708 (was 4862-6364), heap phase 4749 / 4729 / 5427 (was 4040-5804); spills 72 + 95 -> 36 + 67; + TEAM=16 wall 715 / 759 (unchanged, balance-bound); tile 659 ms (unchanged). The engine's k6 goes + ~0.61x -> ~0.80x for free; the 4.0 ms mode is not reached - the remaining spills are the i16 + chains and broadcasts. NEXT: the same transpose for k5's qh and k3's hmask (eight sites per byte, + `lshr s` + `and 1`, retires the k3 pin) - the memo's other two high-bit-plane losers. 3. Noise: the one-thread bench and the reference both drift ~5-8% run to run; interleaved rounds hold ours steady, the reference is re-run per table. iq3xxs/iq2s/iq2xxs tie at 1.00; k2 now clear. From 7211c6f23948a66df42828b2215965bcffbe0e44 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 07:30:24 -0700 Subject: [PATCH 034/123] k3 planes, CPU flavor: qs packed per sub-block at 2j and hmask one column per sub-block at bit s - the volatile pin retires k3's qs (four sub-blocks per GGUF byte) and hmask (eight per byte) were copied verbatim into the grp planes, so the gemv shared 24 column loads across a sub-block group and needed the volatile re-load pin to stop CSE spilling them - decode shape only, the tile kept the liveness. Column 2blk + half now holds a sub-block's four j sites at 2j and hmask column blk its eight sites (lo at j, hi at 4 + j): three loads per sub-block, uniform shifts, nothing lives past its sub-block, same 96 bytes. The same six readers as k6's move (repack_k3_grp, the gemv/tile arm, dequant_k3_row_grp, k3_grp_row_dot, the layout inverse; k3 tiles read the planes directly, no byte-expand); pin_reload is gone. IMAGE_VERSION 28, LLVM_JIT_CODEGEN_VERSION 0x5c. TEST 90/90; test_repack, test_kquant, test_gguf_quant green. zen2 one thread, us: k3 decode 2590 / 2925 / 3012 heap phase, 2555 / 2825 / 3504 page-aligned, against 2648-2850 with the pin (reference 2701); the tile 733 -> 686 ms. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 32 +++++-------------- modules/dasLLAMA/dasllama/dasllama_image.das | 2 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 17 ++++++++-- .../dasllama/dasllama_math_default.das | 6 ++-- .../dasLLAMA/dasllama/dasllama_math_gen.das | 6 ++-- modules/dasLLAMA/dasllama/dasllama_repack.das | 21 ++++++++---- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- 7 files changed, 43 insertions(+), 43 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index e78f09e556..6aa994b19b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -931,16 +931,6 @@ def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; na return LLVMBuildOr(b, w, sel, name) } -//! k3's decode re-loads the 24 shared qs/hmask column vectors where each sub-block uses them (L1 hits) -//! instead of CSE keeping them live against 16 ymm registers and spilling (3198 -> 2691 us). Volatile is -//! LLVM's "load here"; it also forbids reordering, so the tile shape (token reuse, ILP) keeps CSE. -def private pin_reload(var a, b, c, d : LLVMOpaqueValue?) { - LLVMSetVolatile(a, 1) - LLVMSetVolatile(b, 1) - LLVMSetVolatile(c, 1) - LLVMSetVolatile(d, 1) -} - // One 256-weight SUPERBLOCK, K-quant grp form (te.kq set): weight vectors unpacked // ONCE per (sub-block, dword-group) and dotted against every token — weight-stationary, // bit-exact vs per-token GEMVs (tokCount=1). kq v2 folds integer sub-scales/bsums against Q8_K-form activations. @@ -1043,20 +1033,14 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var wlo : LLVMOpaqueValue? var whi : LLVMOpaqueValue? if (k3) { - // k3: qs column (blk/4)*8 + j (lo) / + 4 (hi), 2-bit lane at shift 2*(blk%4); hmask column j / j+4, bit blk - let c0 = (blk / 4) * 8 + j - let sh = (blk % 4) * 2 - var q3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(c0 * 4 * mr + qd * w8)), ""), "q3l{blk}_{j * rq + qd}") - var q3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((c0 + 4) * 4 * mr + qd * w8)), ""), "q3h{blk}_{j * rq + qd}") - var h3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + j * 4) * mr + qd * w8)), ""), "h3l{blk}_{j * rq + qd}") - var h3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + (j + 4) * 4) * mr + qd * w8)), ""), "h3h{blk}_{j * rq + qd}") - if (tokCount == 1 && rq == 1) { // the decode shape only: the tile's token reuse wants the columns kept live - pin_reload(q3l, q3h, h3l, h3h) - } - var lo2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3l, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") - var hi2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3h, splat_i8w(te, sh), ""), splat_i8w(te, 3), "") - var lob = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3l, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") - var hib = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3h, splat_i8w(te, blk), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") + // k3: qs columns 2blk / 2blk + 1 hold this sub-block's four j sites at 2j, hmask column blk its eight sites (lo at j, hi at 4 + j) - three loads per sub-block, nothing lives past it + var q3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 2) * 4 * mr + qd * w8)), ""), "q3l{blk}_{qd}") + var q3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 2 + 1) * 4 * mr + qd * w8)), ""), "q3h{blk}_{qd}") + var h3 = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + blk * 4) * mr + qd * w8)), ""), "h3{blk}_{qd}") + var lo2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3l, splat_i8w(te, 2 * j), ""), splat_i8w(te, 3), "") + var hi2 = LLVMBuildAnd(b, LLVMBuildLShr(b, q3h, splat_i8w(te, 2 * j), ""), splat_i8w(te, 3), "") + var lob = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3, splat_i8w(te, j), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") + var hib = LLVMBuildShl(b, LLVMBuildAnd(b, LLVMBuildLShr(b, h3, splat_i8w(te, 4 + j), ""), splat_i8w(te, 1), ""), splat_i8w(te, 2), "") wlo = LLVMBuildOr(b, lo2, lob, "wlo3{blk}_{j * rq + qd}") whi = LLVMBuildOr(b, hi2, hib, "whi3{blk}_{j * rq + qd}") } elif (k2) { diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index a595c38c81..642ae1458d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -38,7 +38,7 @@ require dasllama/dasllama_load // WhisperModel.enc) contribute their planes under dotted names ("enc.fblob"); string-array // fields ride the meta blob via serialize_strings — raw string pointers can't be planes. -let IMAGE_VERSION = 27 // k6's qh plane packed per sub-block (CPU flavor) +let IMAGE_VERSION = 28 // k6's qh and k3's qs/hmask planes packed per sub-block (CPU flavor) //! The metal (blob-only) flavor's identity tag: q8 planes ride the 34B block_q8_0 blob and the //! kq scale planes their GPU forms (convert_model_to_metal_blob) — flavors are per-config and diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 4e2dd9cfbd..c2f71d6043 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -968,12 +968,23 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(96l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.k3) { // 16 qs columns, then the 8 hmask columns at 64*mr + } elif (fmt == KqFmt.k3) { // qs columns 2blk + half carry a sub-block's four j sites at 2j; hmask column blk its eight sites at bit s for (m in range64(64l)) { - dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] + let half = (m % 32l) / 16l + let j = (m % 16l) / 4l + var qbyte = 0u + for (bb in range64(4l)) { + let col = ((m / 32l) * 4l + bb) * 2l + half + qbyte |= ((uint(gq[(col * mr + r) * 4l + m % 4l]) >> uint(2l * j)) & 3u) << uint(2l * bb) + } + dq[m] = uint8(qbyte) } for (m in range64(32l)) { - dq[64l + m] = gq[64l * mr + ((m / 4l) * mr + r) * 4l + m % 4l] + var hbyte = 0u + for (blk in range64(8l)) { + hbyte |= ((uint(gq[64l * mr + (blk * mr + r) * 4l + m % 4l]) >> uint(m / 4l)) & 1u) << uint(blk) + } + dq[64l + m] = uint8(hbyte) } } elif (fmt == KqFmt.k2) { // 16 qs columns, nothing else for (m in range64(64l)) { diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 8165697848..07be147cc4 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -1066,14 +1066,12 @@ def private dequant_k3_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, ns let d = f16_to_f32(uint(ksg[sb + 16l * mr + 2l * r]) | (uint(ksg[sb + 16l * mr + 2l * r + 1l]) << 8u)) for (blk in range64(8l)) { let kb = sbi * 256l + blk * 32l - let h = blk / 4l - let sh = int((blk % 4l) * 2l) let s0 = d * float(int(int8(ksg[sb + 2l * blk * mr + r]))) let s1 = d * float(int(int8(ksg[sb + (2l * blk + 1l) * mr + r]))) for (l4 in range64(8l)) { for (t in range64(4l)) { - let q2 = (int(kqg[qb + ((h * 8l + l4) * mr + r) * 4l + t]) >> sh) & 3 - let hb = (int(kqg[qb + 64l * mr + (l4 * mr + r) * 4l + t]) >> int(blk)) & 1 + let q2 = (int(kqg[qb + ((blk * 2l + l4 / 4l) * mr + r) * 4l + t]) >> int((l4 % 4l) * 2l)) & 3 + let hb = (int(kqg[qb + 64l * mr + (blk * mr + r) * 4l + t]) >> int(l4)) & 1 dst[kb + l4 * 4l + t] = (l4 < 4l ? s0 : s1) * float((q2 | (hb << 2)) - 4) } } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index ef5b799629..9f05c22cd8 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -404,14 +404,12 @@ def k3_grp_row_dot(kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : var bsum = 0 for (blk in range64(8l)) { let b = sbi * 8l + blk - let h = blk / 4l - let sh = int((blk % 4l) * 2l) var ilo = 0 var ihi = 0 for (l4 in range64(8l)) { for (t in range64(4l)) { - let q2 = (int(kqg[qb + ((h * 8l + l4) * mr + r) * 4l + t]) >> sh) & 3 - let hb = (int(kqg[qb + 64l * mr + (l4 * mr + r) * 4l + t]) >> int(blk)) & 1 + let q2 = (int(kqg[qb + ((blk * 2l + l4 / 4l) * mr + r) * 4l + t]) >> int((l4 % 4l) * 2l)) & 3 + let hb = (int(kqg[qb + 64l * mr + (blk * mr + r) * 4l + t]) >> int(l4)) & 1 let q = q2 | (hb << 2) if (l4 < 4l) { ilo += q * int(xqp[b * 32l + l4 * 4l + t]) diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 14675c2ddb..b34b98833a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -448,14 +448,23 @@ def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { for (r in range64(mr)) { let sq = (g * mr + r) * qrow + sbi * 96l let ss = (g * mr + r) * srow + sbi * 18l - for (c in range64(16l)) { - for (t in range64(4l)) { - kq[dq + (c * mr + r) * 4l + t] = tqp[sq + c * 4l + t] + for (blk in range64(8l)) { // qs: column 2blk + half holds the sub-block's four j sites at 2j; hmask: column blk, its eight sites at bit s (lo j, hi 4 + j) + for (half in range64(2l)) { + for (t in range64(4l)) { + var qbyte = 0u + for (j in range64(4l)) { + let l = (blk / 4l) * 32l + half * 16l + j * 4l + t + qbyte |= ((uint(tqp[sq + l]) >> uint(2l * (blk % 4l))) & 3u) << uint(2l * j) + } + kq[dq + ((blk * 2l + half) * mr + r) * 4l + t] = uint8(qbyte) + } } - } - for (c in range64(8l)) { for (t in range64(4l)) { - kq[dq + 64l * mr + (c * mr + r) * 4l + t] = tqp[sq + 64l + c * 4l + t] + var hbyte = 0u + for (sidx in range64(8l)) { + hbyte |= ((uint(tqp[sq + 64l + sidx * 4l + t]) >> uint(blk)) & 1u) << uint(sidx) + } + kq[dq + 64l * mr + (blk * mr + r) * 4l + t] = uint8(hbyte) } } for (idx in range64(16l)) { diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 69e52799bd..8e8ed3846b 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5bul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5cul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) From 02af2f3b253fd4134509fac97521327b9c656179 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 07:32:00 -0700 Subject: [PATCH 035/123] kernel parity plan: the k3 transpose landing and the k5 shape --- plans/kernel_parity_pass.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 02e6dabf68..487c3fa61d 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -269,8 +269,16 @@ k6 0.90x, k5 0.91x): 4708 (was 4862-6364), heap phase 4749 / 4729 / 5427 (was 4040-5804); spills 72 + 95 -> 36 + 67; TEAM=16 wall 715 / 759 (unchanged, balance-bound); tile 659 ms (unchanged). The engine's k6 goes ~0.61x -> ~0.80x for free; the 4.0 ms mode is not reached - the remaining spills are the i16 - chains and broadcasts. NEXT: the same transpose for k5's qh and k3's hmask (eight sites per byte, - `lshr s` + `and 1`, retires the k3 pin) - the memo's other two high-bit-plane losers. + chains and broadcasts. + LANDED k3 (a387ed230): qs per sub-block at 2j (two loads) and hmask one column per sub-block at bit + s (one load) - three loads per sub-block, the volatile pin gone, same 96 bytes; IMAGE_VERSION 28, JIT + 0x5c; TEST 90/90 + the three module tests green. zen2 one thread, us: decode 2590 / 2925 / 3012 + heap phase, 2555 / 2825 / 3504 page-aligned, against 2648-2850 with the pin (reference 2701) - + equal at one thread with a wider draw; the TILE 733 -> 686 ms (the pin was decode-only, the layout + is not). Body 1027 instructions, 41 + 39 spill ops (was 1152, 42 + 13 with the pin). k5 is already + site-major (one qh byte per row per site) - its cost is the byte -> lane expansion (shuffle + two + masked tests per site); the k3 hmask shape fits it bit for bit: one column per sub-block, deposit + `(hb << (4 - j)) & 0x10` / `(hb >> j) & 0x10` - six ops per site against ten plus a shuffle. NEXT. 3. Noise: the one-thread bench and the reference both drift ~5-8% run to run; interleaved rounds hold ours steady, the reference is re-run per table. iq3xxs/iq2s/iq2xxs tie at 1.00; k2 now clear. From a166a656a295661254aae61c8d656e9c9cbdd617 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 07:39:58 -0700 Subject: [PATCH 036/123] kernel parity plan: the k5 transpose measured and killed - x86 byte-vector shifts are emulated, the mask test stays --- plans/kernel_parity_pass.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 487c3fa61d..ee461b2135 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -278,7 +278,14 @@ k6 0.90x, k5 0.91x): is not). Body 1027 instructions, 41 + 39 spill ops (was 1152, 42 + 13 with the pin). k5 is already site-major (one qh byte per row per site) - its cost is the byte -> lane expansion (shuffle + two masked tests per site); the k3 hmask shape fits it bit for bit: one column per sub-block, deposit - `(hb << (4 - j)) & 0x10` / `(hb >> j) & 0x10` - six ops per site against ten plus a shuffle. NEXT. + `(hb << (4 - j)) & 0x10` / `(hb >> j) & 0x10` - six ops per site against ten plus a shuffle. + DONE-KILLED k5 (2026-09-01): the transpose was correct (TEST 90/90, the three tests green) and + SLOWER - decode 4378 / 4414 / 4556 us heap phase, 4007 / 4664 / 5212 page-aligned, against + 3590-3760 before (reference 3307); tile 515 -> 559 ms. x86 has no byte-vector shift: `shl` / + `lshr` on <32 x i8> lower to word shifts plus masks, so the per-lane mask test (no shift) was the + cheaper deposit all along; k5's site-major byte and its shuffle stay. Reverted, nothing committed. + The transposes are done: k6 and k3 landed, k5 was already right. On the M1 (sdot lattice, same + IR): k6 1777 -> 1701, k3 1925 -> 1873 us, TEST 90/90. 3. Noise: the one-thread bench and the reference both drift ~5-8% run to run; interleaved rounds hold ours steady, the reference is re-run per table. iq3xxs/iq2s/iq2xxs tie at 1.00; k2 now clear. From 1605d3e262969ae7b53e93bb003ba649cbc917be Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 07:50:05 -0700 Subject: [PATCH 037/123] kq_kernel_bench --team: the engine's own splitter and dispatcher, not a hand-made chunk count The team arm now runs the decode GEMV exactly as dasllama_math does - maybe_parallel_for_indexed over the rows with matmul_chunks_gemv's chunk count (64 chunks of 64 rows at 16 lanes here, self-served) - and prints the split it got; --team is a flag, the lane count is DAS_JOBQUE_THREADS (kernel_ladder.sh TEAM=N). The per-slot column becomes the summed call time of every chunk that slot took. zen2, 16 lanes, engine phase: k6 wall 648 / 760 us against the reference's 442, k4 366 / 474 against 248, slot sums 204 -> 862 - the aggregate 74-87 GB/s where the reference pulls 108-133. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../benchmarks/matmul/kq_kernel_bench.das | 30 +++++++++---------- modules/dasLLAMA/harness/kernel_ladder.sh | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 6cd56b7bac..6b2570f3d6 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -7,6 +7,7 @@ require daslib/clargs require daslib/enum_trait require daslib/fio require daslib/jobque_boost +require dasllama/dasllama_par require dasllama/dasllama_math_gen require dasllama/dasllama_math require dasllama/dasllama_kqformat @@ -50,8 +51,8 @@ struct BenchArgs { base_align : int = 64 @clarg_doc = "Bytes added past the aligned arena base (with --base-align 4096: the planes' phase inside the page)" base_offset : int = 0 - @clarg_doc = "GEMV the engine's way: N row chunks self-served by the jobque team (DAS_JOBQUE_THREADS lanes), wall = the slowest lane, the lanes' fastest and slowest call in two extra columns; 0 = one raw call on this thread" - team : int = 0 + @clarg_doc = "GEMV the engine's way: the engine's own splitter (matmul_chunks_gemv) and dispatcher (maybe_parallel_for_indexed) over DAS_JOBQUE_THREADS lanes, wall = the slowest lane, the lanes' fastest and slowest summed call time in two extra columns; 0 = one raw call on this thread" + team : bool } typedef GemvFn = function<(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void> @@ -250,14 +251,12 @@ struct RowStat { lane_hi : int64 // --team: the slowest } -let TEAM_UNIT = 32l // --team chunk edges land on 32-row boundaries - every registry mr divides them - -//! the engine's dispatch shape: `chunks` row ranges self-served by the team's lanes, each lane adding its call time to laneP[slot] -def team_gemv(fn : GemvFn; units, chunks : int; var yp : float?; kqp, ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64; var laneP : int64?) { - team_parallel_for_indexed(0, units, chunks) <| @(slot : int; jb : int; je : int) { +//! the engine's decode dispatch, verbatim (dasllama_math's matmul shape): the splitter's chunk count over the +//! rows, the team self-serving them; each lane adds its call time to laneUs[slot] +def team_gemv(fn : GemvFn; var yp : float?; kqp, ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64; var laneP : int64?) { + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { let l0 = ref_time_ticks() - var y = yp - invoke(fn, y, kqp, ksp, xqp, xsp, xbsp, n, int64(jb) * TEAM_UNIT, int64(je) * TEAM_UNIT) + invoke(fn, yp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) unsafe { laneP[slot] += int64(get_time_usec(l0)) } @@ -340,15 +339,14 @@ def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va let xsp = addr(p.arena[p.xs]) let xbsp = addr(p.arena[p.xbs]) var yp = addr(p.arena[p.y]) - if (cfg.team > 0) { - let units = int(d / TEAM_UNIT) + if (cfg.team) { var laneUs : array laneUs |> resize(get_total_hw_jobs() + 1) var laneP = addr(laneUs[0]) for (st, fn in stats, fns) { st.us |> reserve(cfg.rounds) st.lane_lo = 999999999l - team_gemv(fn, units, cfg.team, yp, kqp, ksp, xqp, xsp, xbsp, n, laneP) // warmup + codegen, unmeasured + team_gemv(fn, yp, kqp, ksp, xqp, xsp, xbsp, n, d, laneP) // warmup + codegen, unmeasured } for (_r in range(cfg.rounds)) { for (st, fn in stats, fns) { @@ -356,7 +354,7 @@ def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va u = 0l } let t0 = ref_time_ticks() - team_gemv(fn, units, cfg.team, yp, kqp, ksp, xqp, xsp, xbsp, n, laneP) + team_gemv(fn, yp, kqp, ksp, xqp, xsp, xbsp, n, d, laneP) st.us |> push(int64(get_time_usec(t0))) for (u in laneUs) { if (u > 0l) { @@ -602,12 +600,14 @@ def main : int { var fs <- formats_of(cfg.fmt) if (!cfg.tsv) { print("kq_kernel_bench: n={cfg.n} d={cfg.d} ntok={cfg.ntok} rounds={cfg.rounds} team={cfg.team} tune_mode={get_env_variable("DAS_TUNE_MODE")}\n") - print("fmt\ttier\tperm\tn\td\tntok\tbest_us\tmedian_us\tns_per_sb\tGB/s\tGFLOP/s{cfg.team > 0 ? "\tlane_lo\tlane_hi" : ""}\n") + print("fmt\ttier\tperm\tn\td\tntok\tbest_us\tmedian_us\tns_per_sb\tGB/s\tGFLOP/s{cfg.team ? "\tlane_lo\tlane_hi" : ""}\n") } var rows = 0 - if (cfg.team > 0) { + if (cfg.team) { with_job_que() { setup_dasllama_jobque_() + let chunks = matmul_chunks_gemv(cfg.d, 1, int64(cfg.n) * int64(cfg.d)) + print("# team: {get_total_hw_jobs() + 1} lanes, the engine's splitter gives {chunks} chunks of {cfg.d / chunks} rows\n") rows = run_formats(cfg, fs) } } else { diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh index a1606ab941..4d64d3cb01 100644 --- a/modules/dasLLAMA/harness/kernel_ladder.sh +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -26,7 +26,7 @@ NTOK=${NTOK:-512} ROUNDS=${ROUNDS:-5} TEAM=${TEAM:-0} THREADS=$(( TEAM > 0 ? TEAM : 1 )) -TEAM_ARGS=""; [ "$TEAM" -gt 0 ] && TEAM_ARGS="--team $TEAM" +TEAM_ARGS=""; [ "$TEAM" -gt 0 ] && TEAM_ARGS="--team" WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT From 3b67227fe41ff073e1ae88aac5852fc082137a68 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 07:53:51 -0700 Subject: [PATCH 038/123] kernel ladder BIG=1: the DRAM-bound decode row (d=32768) - the many-lane comparison test-backend-ops' repeated 4096-row op cannot give At 16 and 32 threads the reference reads its 33 MB m=4096 weight out of the 3990X's 256 MB L3 (295 GB/s at 32 threads, three times the box's DRAM), so a many-lane ratio taken at that shape flatters it; a model's weights never stay in L3 between tokens. BIG=1 runs both sides at d=32768 (k4 264 MB), the reference given the m=32768 perf row (a two-line edit in make_test_cases_perf, documented beside the thread define). Measured through the engine's own splitter: k4 16 lanes 4560 vs 4549 us, 32 lanes 4117 vs 5147, k6 16 lanes 6534 vs 6474 - parity at the engine's shape. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_GET_SIDECAR.md | 6 ++++++ modules/dasLLAMA/harness/kernel_ladder.sh | 11 ++++++++--- plans/kernel_parity_pass.md | 16 ++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md index a9adb4da38..6068393124 100644 --- a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -88,6 +88,12 @@ cmake --build build-clean-cpu --target test-backend-ops llama-bench -j 16 `build-clean-cpu` is the flavor the tables name: no GPU backend, no BLAS - a GPU build's `llama-bench -ngl 0` is not CPU-only. +The perf list has no decode row above 4096 x 14336, and on a box with a large L3 that op stays cached +between the repeated runs (the 3990X reads it at 295 GB/s with 32 threads, three times its DRAM) - a +many-lane ratio taken there flatters the reference. For the DRAM-bound row (`BIG=1` in the ladder, +d=32768) add, inside the same `for (int bs ...)` loop in `make_test_cases_perf`, after the 4096 line: +`if (bs == 1) test_cases.emplace_back(new test_mul_mat(type_a, type_b, 32768, bs, 14336, {1, 1}, {1, 1}));` + Confirm the define is in the binary the ladder gets: `grep -c GGML_BENCH_THREADS build-clean-cpu/bin/test-backend-ops` prints 1. A box with several llama.cpp builds is where this bites - a binary that prints 0 runs every core and the ratio column lies; `kernel_ladder.sh` refuses it. diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh index 4d64d3cb01..ef99381cfe 100644 --- a/modules/dasLLAMA/harness/kernel_ladder.sh +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -11,8 +11,11 @@ # LCPP_TBO test-backend-ops (default: $HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe) # NTOK prefill tokens (default 512; 0 = decode rows only) # ROUNDS interleaved rounds per row (default 5) -# TEAM lanes: the bench dispatches the GEMV the engine's way (--team N, DAS_JOBQUE_THREADS=N) +# TEAM lanes: the bench dispatches the GEMV the engine's way (--team, DAS_JOBQUE_THREADS=N) # and the reference runs GGML_BENCH_THREADS=N; unset = one thread, one raw call +# BIG=1 the decode row at d=32768 rows (a weight above any L3 - the many-lane comparison that +# test-backend-ops perf's repeated 4096-row op does not give; the reference needs the +# m=32768 perf case, a two-line edit beside the thread define - HOW_TO_GET_SIDECAR.md) # Output: a TSV table on stdout - fmt tier perm ours_us ours_med_us ref_us ratio - plus the box line. set -e -o pipefail # a failing daslang must not hide behind the join ROOT=$(cd "$(dirname "$0")/../../.." && pwd) # the tree this script lives in, whatever the env says @@ -25,6 +28,8 @@ FMTS=${1:-all} NTOK=${NTOK:-512} ROUNDS=${ROUNDS:-5} TEAM=${TEAM:-0} +BIG=${BIG:-0} +ROWS=$(( BIG > 0 ? 32768 : 4096 )) THREADS=$(( TEAM > 0 ? TEAM : 1 )) TEAM_ARGS=""; [ "$TEAM" -gt 0 ] && TEAM_ARGS="--team" WORK=$(mktemp -d) @@ -47,7 +52,7 @@ echo "# kernel ladder $(date +%F) box=$(hostname) daslang=$DASLANG ref=$TBO grep -q GGML_BENCH_THREADS "$TBO" || { echo "kernel_ladder: '$TBO' lacks the GGML_BENCH_THREADS define - it would run every core and the ratio column would lie (harness README)" >&2; exit 1; } set +e DAS_TUNE_MODE=normal DAS_JOBQUE_THREADS=$THREADS "$DASLANG" -jit "$ROOT/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das" \ - -- --fmt "$FMTS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv $TEAM_ARGS > "$WORK/ours.raw" 2> "$WORK/ours.err" + -- --fmt "$FMTS" --d "$ROWS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv $TEAM_ARGS > "$WORK/ours.raw" 2> "$WORK/ours.err" rc=$? set -e awk -F'\t' 'NF >= 11' "$WORK/ours.raw" > "$WORK/ours.tsv" || true # the TSV rows; engine and tune notes fall away @@ -62,7 +67,7 @@ fi for f in $(cut -f1 "$WORK/ours.tsv" | sort -u); do t=$(ggml_type "$f") [ -n "$t" ] || { echo "kernel_ladder: no ggml type for '$f'" >&2; continue; } - GGML_BENCH_THREADS=$THREADS "$TBO" perf -b CPU -o MUL_MAT -p "type_a=$t,type_b=f32,m=4096,n=(1|512)," 2>/dev/null \ + GGML_BENCH_THREADS=$THREADS "$TBO" perf -b CPU -o MUL_MAT -p "type_a=$t,type_b=f32,m=$ROWS,n=(1|512)," 2>/dev/null \ | sed 's/\x1b\[[0-9;]*m//g' | awk -v f="$f" ' /MUL_MAT\(/ { n = $0; sub(/.*,n=/, "", n); sub(/,.*/, "", n) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index ee461b2135..e540d21838 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -286,6 +286,22 @@ k6 0.90x, k5 0.91x): cheaper deposit all along; k5's site-major byte and its shuffle stay. Reverted, nothing committed. The transposes are done: k6 and k3 landed, k5 was already right. On the M1 (sdot lattice, same IR): k6 1777 -> 1701, k3 1925 -> 1873 us, TEST 90/90. + +The 16-lane row and what it measures (2026-09-01, `--team`, the engine's own splitter: 64 self-served +chunks of 64 rows): k4 wall against the reference at the same thread count - 8 lanes 648 vs 675 us +(1.04x), 16 lanes 469 vs 341 (0.73x), 32 lanes 342 vs 112 (0.33x); k6 at 8 lanes 1165 vs 1041 +(0.89x). The reference at 32 threads moves 33 MB in 112 us = 295 GB/s, three times this box's DRAM: +`test-backend-ops perf` repeats one small op and its row -> thread mapping is stable, so the weights +stay in the 3990X's 256 MB of L3 (16 MB per CCX); our self-served chunks move rows between CCX slices +every call and pay DRAM. At 8 lanes, where neither side gets the L3, we are at parity. A model's +weights (4-30 GB) never stay in L3 between tokens, so the fair many-lane comparison is a working set +above L3 - d=32768 (k4 264 MB) - measured next. If parity holds there, the many-lane gap seen at +m=4096 is the microbench's artifact, not the dispatch. +MEASURED (the reference given the m=32768 perf row - HOW_TO_GET_SIDECAR; the ladder's `BIG=1`): +k4 16 lanes 4560 vs 4549 us (1.00x), 32 lanes 4117 vs 5147 (1.25x); k6 16 lanes 6534 vs 6474 +(0.99x) - both sides at the box's ~58-64 GB/s DRAM rate. PARITY at the engine's shape; the +"dispatch grain" question is closed - nothing to fix in the splitter. The m=4096 many-lane rows +stay in the ladder as a kernel-in-L3 view only; the BIG row is the one that speaks for decode. 3. Noise: the one-thread bench and the reference both drift ~5-8% run to run; interleaved rounds hold ours steady, the reference is re-run per table. iq3xxs/iq2s/iq2xxs tie at 1.00; k2 now clear. From 04d54f57e6be90cc0adfa4eef08e2dc382122de7 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:04:48 -0700 Subject: [PATCH 039/123] k6/k3 decode, x86: the per-16 sub-scale rides the chain flush - pmaddwd(chain, scale pair splat) straight into the accumulator The k63 fold widened each i16 chain with pmaddwd against ones, then multiplied the i32 result by the row's sub-scale (four vpmulld per sub-block) and added into iacc through a0/a1. The widening can carry the scale: pmaddwd(chain_i16, s16) where s16 is the rows' int8 sub-scales sign-extended and duplicated into each row's two lanes - exact in i32 (|chain| <= 32004, |scale| <= 128, sixteen flushes per superblock) - so the flush lands in iacc directly, the multiplies and the a0/a1 accumulators are gone, and the bsum term keeps its two products. k6 body 1161 -> 1155 instructions, 32 -> 16 vpmulld, spills 36 + 67 -> 31 + 52. zen2 one thread, us: k6 decode at the engine phase 3527 / 3636 / 3851 (was 4444-4708; reference 3683), heap phase 3808 / 4007; k3 2488 / 2613 / 2893 (reference 2701). Tiles within their noise band. gen_tune_probe TEST 90/90. The sidecar how-to gains the many-lane DRAM-bound ladder row. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_GET_SIDECAR.md | 6 ++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 61 ++++++++++++++++--- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- plans/kernel_parity_pass.md | 7 +++ 4 files changed, 65 insertions(+), 11 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md index 6068393124..7c3930f47e 100644 --- a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -116,6 +116,12 @@ LCPP_TBO=~/llama.cpp/build-clean-cpu/bin/test-backend-ops NTOK=512 ROUNDS=5 \ bash modules/dasLLAMA/harness/kernel_ladder.sh all > ladder.tsv 2> ladder.err ``` +Then the engine's shape - every lane, a weight above the box's L3 (needs the m=32768 perf row, section 3): + +``` +LCPP_TBO=~/llama.cpp/build-clean-cpu/bin/test-backend-ops NTOK=0 ROUNDS=5 TEAM=16 BIG=1 bash modules/dasLLAMA/harness/kernel_ladder.sh all > ladder_big16.tsv 2> ladder_big16.err +``` + One thread, no model: every format's stamped kernel against the reference exe's `test-backend-ops perf` at the same shape, decode and prefill rows, ratio = reference / ours. This is the box's table; it goes into `plans/kernel_parity_pass.md`'s fact base (and the records store once the kernel board exists). diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 6aa994b19b..bb9f717664 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -886,6 +886,24 @@ def private madd16_flush(var te : TileEmit; var acc, chain : LLVMOpaqueValue?) : return LLVMBuildAdd(b, acc, quads, "a32") } +//! the rows' int8 sub-scales as an i16 pair splat (lanes 2r, 2r + 1 = row r): pmaddwd against a chain widens AND scales in one op +def private scale16_dup(var te : TileEmit; var off : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { + let b = te.builder + var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) + var p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, off, "") + var s16 = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, p, 1u, ""), te.vni16, "") + let dup <- [for (i in range(2 * te.rv)); i / 2] + return LLVMBuildShuffleVector(b, te.types, s16, s16, dup, name) +} + +//! the chain flush with the sub-scale folded in - exact in i32 (|chain| <= 32004, |scale| <= 128, 16 flushes per superblock) +def private madd16_flush_scaled(var te : TileEmit; var acc, chain, s16 : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + var wargs <- [chain, s16] + var quads = LLVMBuildCall2(b, te.maddwd_ty, te.maddwd_decl, wargs, "squads") + return LLVMBuildAdd(b, acc, quads, "a32s") +} + // one row-byte load at BYTE offset, each byte splatted across its row's 4 lanes to // vwi8 (constant shufflevector — the backend picks tbl/pshufb itself; no intrinsic needed) def private load_row_bytes_x4(var te : TileEmit; var base, off : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { @@ -959,6 +977,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f // x64 dot kinds take the activation dword straight from memory (vpbroadcastd, see kq_dot_mem); NEON keeps chunk loads let memBcast = te.dotKind != DOT_SDOT && te.dotKind != DOT_SMMLA let madd16 = memBcast && te.dotKind == DOT_MADDUBS + let fuseScale = madd16 && k63 // k6/k3 decode: the per-16 sub-scale rides the chain flush (pmaddwd), no i32 multiply, no a0/a1 var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) // weight-side superblock scale rows: k4/k5 f16 d/dmin; k6/k3 f16 d only; q40 has none (per-32 d loads live below) var dv : LLVMOpaqueValue? [2] @@ -1026,6 +1045,14 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f p16lo[i] = null p16hi[i] = null } + var s16lo : LLVMOpaqueValue? [2] // fuseScale: this sub-block's two sub-scales as i16 pair splats + var s16hi : LLVMOpaqueValue? [2] + if (fuseScale) { + for (qd in range(rq)) { + s16lo[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * blk * mr + qd * te.rv)), ""), "s16{blk}_{qd}a") + s16hi[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), "s16{blk}_{qd}b") + } + } for (j in range(4)) { for (qd in range(rq)) { // weight vectors unpacked ONCE, dotted against every token below (kqBytes panel loads verbatim instead) @@ -1110,23 +1137,35 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f // i16 chain flushes at the format's overflow bound: k5 after 4 madds, k6 after 2 (2 x 16002); wholeSub formats run the sub-block if (madd16 && (j == 1 || j == 3) && !wholeSub) { for (k in range(tokCount * rq)) { - a0[k] = madd16_flush(te, a0[k], p16lo[k]) - p16lo[k] = null - if (per16) { - a1[k] = madd16_flush(te, a1[k], p16hi[k]) + if (fuseScale) { + iacc[k] = madd16_flush_scaled(te, iacc[k], p16lo[k], s16lo[k % rq]) + iacc[k] = madd16_flush_scaled(te, iacc[k], p16hi[k], s16hi[k % rq]) p16hi[k] = null + } else { + a0[k] = madd16_flush(te, a0[k], p16lo[k]) + if (per16) { + a1[k] = madd16_flush(te, a1[k], p16hi[k]) + p16hi[k] = null + } } + p16lo[k] = null } } } if (madd16 && wholeSub) { // one widen per (token, qd, chain) for the whole sub-block for (k in range(tokCount * rq)) { - a0[k] = madd16_flush(te, a0[k], p16lo[k]) - p16lo[k] = null - if (per16) { - a1[k] = madd16_flush(te, a1[k], p16hi[k]) + if (fuseScale) { + iacc[k] = madd16_flush_scaled(te, iacc[k], p16lo[k], s16lo[k % rq]) + iacc[k] = madd16_flush_scaled(te, iacc[k], p16hi[k], s16hi[k % rq]) p16hi[k] = null + } else { + a0[k] = madd16_flush(te, a0[k], p16lo[k]) + if (per16) { + a1[k] = madd16_flush(te, a1[k], p16hi[k]) + p16hi[k] = null + } } + p16lo[k] = null } } // the integer fold — no float ops per block, except q40 (its per-32 f16 d admits no cross-block integer fold) @@ -1191,9 +1230,11 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var bs0v = splat_i32(te, bs0, "bs0v{tk}_{blk}") var bs1v = splat_i32(te, bs1, "bs1v{tk}_{blk}") for (qd in range(rq)) { - var ia = LLVMBuildAdd(b, LLVMBuildMul(b, scv[qd], a0[i * rq + qd], ""), LLVMBuildMul(b, mnv[qd], a1[i * rq + qd], ""), "ia6{tk}_{blk}_{qd}") + if (!fuseScale) { // the fused flushes already scaled the dots into iacc + var ia = LLVMBuildAdd(b, LLVMBuildMul(b, scv[qd], a0[i * rq + qd], ""), LLVMBuildMul(b, mnv[qd], a1[i * rq + qd], ""), "ia6{tk}_{blk}_{qd}") + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tk}_{blk}_{qd}") + } var ba = LLVMBuildAdd(b, LLVMBuildMul(b, scv[qd], bs0v, ""), LLVMBuildMul(b, mnv[qd], bs1v, ""), "ba6{tk}_{blk}_{qd}") - iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], ia, "ia{tk}_{blk}_{qd}") bacc[i * rq + qd] = LLVMBuildAdd(b, bacc[i * rq + qd], ba, "ba{tk}_{blk}_{qd}") } } elif (k2) { diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 8e8ed3846b..4bf413a987 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5cul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5dul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index e540d21838..5bac8a5928 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -286,6 +286,13 @@ k6 0.90x, k5 0.91x): cheaper deposit all along; k5's site-major byte and its shuffle stay. Reverted, nothing committed. The transposes are done: k6 and k3 landed, k5 was already right. On the M1 (sdot lattice, same IR): k6 1777 -> 1701, k3 1925 -> 1873 us, TEST 90/90. + LANDED the fused scale flush (x86 maddubs lattice, k6/k3): the per-16 sub-scale rides the chain's + widening - `pmaddwd(chain_i16, scale_i16_pair_splat)` straight into iacc, exact in i32 (|chain| <= + 32004 x |scale| <= 128) - instead of pmaddwd(ones) + i32 vpmulld into a0/a1 and a second add. 32 -> + 16 vpmulld, a0/a1 gone from the live set, k6 spills 36 + 67 -> 31 + 52. zen2 k6 decode at the + engine phase 3527 / 3636 / 3851 us against the reference's 3683 (was 4444-4708): PARITY; heap 3808 / + 4007. k3 engine phase 2488 / 2613 / 2893 (reference 2701, was 2555-3504). Tiles unchanged within + noise (k6 588-702 across runs, k3 660). TEST 90/90. The M1 is untouched (sdot has no i16 chain). The 16-lane row and what it measures (2026-09-01, `--team`, the engine's own splitter: 64 self-served chunks of 64 rows): k4 wall against the reference at the same thread count - 8 lanes 648 vs 675 us From 1880c43c769ac0cfb77ab9cc885fe73432f80eda Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:09:36 -0700 Subject: [PATCH 040/123] k4/k5 decode, x86: the sub-scale rides the chain flush too - one 6-bit scale per sub-block, zero-extended into the pair splat The same fusion as k6/k3: pmaddwd(chain_i16, scale pair splat) straight into iacc replaces pmaddwd(ones) + vpmulld + the a0/a1 accumulators; k4/k5 have one chain and one unsigned 6-bit scale per sub-block (|chain| <= 31496 x 63 fits i32 with room). gen_tune_probe TEST 90/90. zen2 one thread at the engine phase, us: k5 decode 3399 / 3474 / 3511 (reference 3307), k4 2030 / 2106 / 2151 (reference 2435); tiles k5 505, k4 482 ms - all within their bands. LLVM_JIT_CODEGEN_VERSION 0x5e. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 37 ++++++++++++------- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- plans/kernel_parity_pass.md | 5 +++ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index bb9f717664..b9a2669469 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -886,12 +886,13 @@ def private madd16_flush(var te : TileEmit; var acc, chain : LLVMOpaqueValue?) : return LLVMBuildAdd(b, acc, quads, "a32") } -//! the rows' int8 sub-scales as an i16 pair splat (lanes 2r, 2r + 1 = row r): pmaddwd against a chain widens AND scales in one op -def private scale16_dup(var te : TileEmit; var off : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { +//! the rows' sub-scales (k6/k3 int8, k4/k5 unsigned 6-bit) as an i16 pair splat (lanes 2r, 2r + 1 = row r): pmaddwd against a chain widens AND scales in one op +def private scale16_dup(var te : TileEmit; var off : LLVMOpaqueValue?; signed : bool; name : string) : LLVMOpaqueValue? { let b = te.builder var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) var p = LLVMBuildGEP2(b, te.types.t_int8, te.sg, off, "") - var s16 = LLVMBuildSExt(b, LLVMBuildLoad2Aligned(b, vri8, p, 1u, ""), te.vni16, "") + var s8 = LLVMBuildLoad2Aligned(b, vri8, p, 1u, "") + var s16 = signed ? LLVMBuildSExt(b, s8, te.vni16, "") : LLVMBuildZExt(b, s8, te.vni16, "") let dup <- [for (i in range(2 * te.rv)); i / 2] return LLVMBuildShuffleVector(b, te.types, s16, s16, dup, name) } @@ -977,7 +978,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f // x64 dot kinds take the activation dword straight from memory (vpbroadcastd, see kq_dot_mem); NEON keeps chunk loads let memBcast = te.dotKind != DOT_SDOT && te.dotKind != DOT_SMMLA let madd16 = memBcast && te.dotKind == DOT_MADDUBS - let fuseScale = madd16 && k63 // k6/k3 decode: the per-16 sub-scale rides the chain flush (pmaddwd), no i32 multiply, no a0/a1 + let fuseScale = madd16 && (k63 || te.kq == 4 || te.kq == 5) // the sub-scale rides the chain flush (pmaddwd), no i32 multiply, no a0/a1 var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) // weight-side superblock scale rows: k4/k5 f16 d/dmin; k6/k3 f16 d only; q40 has none (per-32 d loads live below) var dv : LLVMOpaqueValue? [2] @@ -1047,10 +1048,14 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } var s16lo : LLVMOpaqueValue? [2] // fuseScale: this sub-block's two sub-scales as i16 pair splats var s16hi : LLVMOpaqueValue? [2] - if (fuseScale) { + if (fuseScale && k63) { for (qd in range(rq)) { - s16lo[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * blk * mr + qd * te.rv)), ""), "s16{blk}_{qd}a") - s16hi[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), "s16{blk}_{qd}b") + s16lo[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * blk * mr + qd * te.rv)), ""), true, "s16{blk}_{qd}a") + s16hi[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), true, "s16{blk}_{qd}b") + } + } elif (fuseScale) { // k4/k5: one 6-bit scale per sub-block, one chain + for (qd in range(rq)) { + s16lo[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), false, "s16{blk}_{qd}") } } for (j in range(4)) { @@ -1139,8 +1144,10 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f for (k in range(tokCount * rq)) { if (fuseScale) { iacc[k] = madd16_flush_scaled(te, iacc[k], p16lo[k], s16lo[k % rq]) - iacc[k] = madd16_flush_scaled(te, iacc[k], p16hi[k], s16hi[k % rq]) - p16hi[k] = null + if (per16) { + iacc[k] = madd16_flush_scaled(te, iacc[k], p16hi[k], s16hi[k % rq]) + p16hi[k] = null + } } else { a0[k] = madd16_flush(te, a0[k], p16lo[k]) if (per16) { @@ -1156,8 +1163,10 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f for (k in range(tokCount * rq)) { if (fuseScale) { iacc[k] = madd16_flush_scaled(te, iacc[k], p16lo[k], s16lo[k % rq]) - iacc[k] = madd16_flush_scaled(te, iacc[k], p16hi[k], s16hi[k % rq]) - p16hi[k] = null + if (per16) { + iacc[k] = madd16_flush_scaled(te, iacc[k], p16hi[k], s16hi[k % rq]) + p16hi[k] = null + } } else { a0[k] = madd16_flush(te, a0[k], p16lo[k]) if (per16) { @@ -1249,8 +1258,10 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } else { for (qd in range(rq)) { - var acc = fuseAcc ? a0[i * rq + qd] : LLVMBuildAdd(b, a0[i * rq + qd], a1[i * rq + qd], "acc{tk}_{blk}_{qd}") - iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], LLVMBuildMul(b, scv[qd], acc, ""), "ia{tk}_{blk}_{qd}") + if (!fuseScale) { // the fused flushes already scaled the dots into iacc + var acc = fuseAcc ? a0[i * rq + qd] : LLVMBuildAdd(b, a0[i * rq + qd], a1[i * rq + qd], "acc{tk}_{blk}_{qd}") + iacc[i * rq + qd] = LLVMBuildAdd(b, iacc[i * rq + qd], LLVMBuildMul(b, scv[qd], acc, ""), "ia{tk}_{blk}_{qd}") + } if (!vecBsums) { bacc[i * rq + qd] = LLVMBuildAdd(b, bacc[i * rq + qd], LLVMBuildMul(b, mnv[qd], a32v, ""), "ba{tk}_{blk}_{qd}") } diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 4bf413a987..a97c4bea63 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5dul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5eul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 5bac8a5928..d802138049 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -293,6 +293,11 @@ k6 0.90x, k5 0.91x): engine phase 3527 / 3636 / 3851 us against the reference's 3683 (was 4444-4708): PARITY; heap 3808 / 4007. k3 engine phase 2488 / 2613 / 2893 (reference 2701, was 2555-3504). Tiles unchanged within noise (k6 588-702 across runs, k3 660). TEST 90/90. The M1 is untouched (sdot has no i16 chain). + Extended to k4/k5 (one 6-bit scale per sub-block, zero-extended): TEST 90/90; k5 decode at the engine + phase 3399 / 3474 / 3511 (reference 3307, 0.94-0.97x; ~3600 before), k4 2030-2151 (reference 2435), + tiles unchanged (k5 505, k4 482 ms). k5's last ~4% is the high-bit deposit itself (shuffle + two + masked tests per site) - the per-sub-block layout lost to byte-shift emulation, the memo's D4 stands: + k5 closes at 0.95x on zen2, 1.06x on the M1. The 16-lane row and what it measures (2026-09-01, `--team`, the engine's own splitter: 64 self-served chunks of 64 rows): k4 wall against the reference at the same thread count - 8 lanes 648 vs 675 us From e538ccff3a72e6ca65eb035d273b1dbe0b856e01 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:28:50 -0700 Subject: [PATCH 041/123] kernel ladder: team + DRAM-bound decode rows are the default, SOLO=1 the one-thread kernel table; the q8 family gets the team arm; k4 leaves the fused flush The ladder's decode rows now run the engine's shape by default - TEAM = the box's hardware threads, BIG=1 (d=32768, a weight above any L3), the arena page-aligned (--base-align 4096, the engine's image phase) - and SOLO=1 gives the one-thread kernel table the tuner's races correspond to. The q8 family (q8, mx4) gains the same team arm as the kq formats through a shared time_rows_team loop; without it their many-lane rows were one raw thread against N (0.34 / 0.24 on zen2), with it 1.04 / 1.05. k4 measured 4% slower with the fused scale flush (one flush per sub-block, the pair-splat shuffles outweigh the one multiply) and returns to the plain fold; k5/k6/k3 keep it. TEST 90/90. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../benchmarks/matmul/kq_kernel_bench.das | 96 +++++++++++++------ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 4 +- modules/dasLLAMA/harness/kernel_ladder.sh | 20 ++-- 3 files changed, 81 insertions(+), 39 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 6b2570f3d6..a34873acaa 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -252,7 +252,7 @@ struct RowStat { } //! the engine's decode dispatch, verbatim (dasllama_math's matmul shape): the splitter's chunk count over the -//! rows, the team self-serving them; each lane adds its call time to laneUs[slot] +//! rows, the team self-serving them; each lane adds its call time to laneP[slot] def team_gemv(fn : GemvFn; var yp : float?; kqp, ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64; var laneP : int64?) { maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { let l0 = ref_time_ticks() @@ -263,6 +263,55 @@ def team_gemv(fn : GemvFn; var yp : float?; kqp, ksp : uint8 const?; xqp : int8 } } +def team_gemv_q8(fn : GemvQ8Fn; var yp : float?; wp : int8 const?; sp : float const?; xqp : int8 const?; xsp : float const?; n, d : int64; var laneP : int64?) { + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { + let l0 = ref_time_ticks() + invoke(fn, yp, wp, sp, xqp, xsp, n, int64(rb), int64(re)) + unsafe { + laneP[slot] += int64(get_time_usec(l0)) + } + } +} + +def team_gemv_mx4(fn : GemvMx4Fn; var yp : float?; wn, we : uint8 const?; xqp : int8 const?; xsp : float const?; n, d : int64; var laneP : int64?) { + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { + let l0 = ref_time_ticks() + invoke(fn, yp, wn, we, xqp, xsp, n, int64(rb), int64(re)) + unsafe { + laneP[slot] += int64(get_time_usec(l0)) + } + } +} + +//! the team arm's interleaved timing: the block dispatches row `row` the engine's way and every lane adds its +//! call time to laneP[slot]; the row keeps the wall (best, median) and the lanes' fastest and slowest sums +def time_rows_team(var stats : array; rounds : int; blk : block<(row : int; var laneP : int64?) : void>) { + var laneUs : array + laneUs |> resize(get_total_hw_jobs() + 1) + var laneP = unsafe(addr(laneUs[0])) + for (row in range(length(stats))) { + stats[row].us |> reserve(rounds) + stats[row].lane_lo = 999999999l + invoke(blk, row, laneP) // warmup + codegen, unmeasured + } + for (_r in range(rounds)) { + for (row in range(length(stats))) { + for (u in laneUs) { + u = 0l + } + let t0 = ref_time_ticks() + invoke(blk, row, laneP) + stats[row].us |> push(int64(get_time_usec(t0))) + for (u in laneUs) { + if (u > 0l) { + stats[row].lane_lo = min(stats[row].lane_lo, u) + stats[row].lane_hi = max(stats[row].lane_hi, u) + } + } + } + } +} + def median_us(var us : array) : int64 { var s := us sort(s) @@ -340,29 +389,8 @@ def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va let xbsp = addr(p.arena[p.xbs]) var yp = addr(p.arena[p.y]) if (cfg.team) { - var laneUs : array - laneUs |> resize(get_total_hw_jobs() + 1) - var laneP = addr(laneUs[0]) - for (st, fn in stats, fns) { - st.us |> reserve(cfg.rounds) - st.lane_lo = 999999999l - team_gemv(fn, yp, kqp, ksp, xqp, xsp, xbsp, n, d, laneP) // warmup + codegen, unmeasured - } - for (_r in range(cfg.rounds)) { - for (st, fn in stats, fns) { - for (u in laneUs) { - u = 0l - } - let t0 = ref_time_ticks() - team_gemv(fn, yp, kqp, ksp, xqp, xsp, xbsp, n, d, laneP) - st.us |> push(int64(get_time_usec(t0))) - for (u in laneUs) { - if (u > 0l) { - st.lane_lo = min(st.lane_lo, u) - st.lane_hi = max(st.lane_hi, u) - } - } - } + time_rows_team(stats, cfg.rounds) $(row : int; var laneP : int64?) { + team_gemv(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, d, laneP) } } else { time_rows(stats, cfg.rounds) $(row : int) { @@ -511,11 +539,21 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes var fnsQ : array var fnsM : array q8_gemv_rows(cfg, mx4, stats, fnsQ, fnsM) - time_rows(stats, cfg.rounds) $(row : int) { - if (mx4) { - invoke(fnsM[row], yp, kqp, ksp, xqp, xsp, n, 0l, d) - } else { - invoke(fnsQ[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, 0l, d) + if (cfg.team) { + time_rows_team(stats, cfg.rounds) $(row : int; var laneP : int64?) { + if (mx4) { + team_gemv_mx4(fnsM[row], yp, kqp, ksp, xqp, xsp, n, d, laneP) + } else { + team_gemv_q8(fnsQ[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, d, laneP) + } + } + } else { + time_rows(stats, cfg.rounds) $(row : int) { + if (mx4) { + invoke(fnsM[row], yp, kqp, ksp, xqp, xsp, n, 0l, d) + } else { + invoke(fnsQ[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, 0l, d) + } } } for (st in stats) { diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index b9a2669469..44f92394fe 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -978,7 +978,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f // x64 dot kinds take the activation dword straight from memory (vpbroadcastd, see kq_dot_mem); NEON keeps chunk loads let memBcast = te.dotKind != DOT_SDOT && te.dotKind != DOT_SMMLA let madd16 = memBcast && te.dotKind == DOT_MADDUBS - let fuseScale = madd16 && (k63 || te.kq == 4 || te.kq == 5) // the sub-scale rides the chain flush (pmaddwd), no i32 multiply, no a0/a1 + let fuseScale = madd16 && (k63 || te.kq == 5) // the sub-scale rides the chain flush (pmaddwd), no i32 multiply, no a0/a1; k4 measured 4% slower fused (one flush per sub-block) var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) // weight-side superblock scale rows: k4/k5 f16 d/dmin; k6/k3 f16 d only; q40 has none (per-32 d loads live below) var dv : LLVMOpaqueValue? [2] @@ -1053,7 +1053,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f s16lo[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(2 * blk * mr + qd * te.rv)), ""), true, "s16{blk}_{qd}a") s16hi[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64((2 * blk + 1) * mr + qd * te.rv)), ""), true, "s16{blk}_{qd}b") } - } elif (fuseScale) { // k4/k5: one 6-bit scale per sub-block, one chain + } elif (fuseScale) { // k5: one 6-bit scale per sub-block, one chain for (qd in range(rq)) { s16lo[qd] = scale16_dup(te, LLVMBuildAdd(b, sb, te.types->ConstI64(uint64(4 * mr + blk * mr + qd * te.rv)), ""), false, "s16{blk}_{qd}") } diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh index ef99381cfe..705d0edfb7 100644 --- a/modules/dasLLAMA/harness/kernel_ladder.sh +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -3,7 +3,8 @@ # one thread, normal tune mode = the winner this box ships) against the reference exe's # test-backend-ops perf at the same shape, decode (n=1) and prefill (n=512) rows, joined with # the ratio (reference us / ours us; >= 1.0 = ours is faster). No model, no jobque: the kernel -# and nothing else. The reference binary needs the GGML_BENCH_THREADS define (harness README). +# and nothing else. The bench's arena sits page-aligned (--base-align 4096) - the engine's image +# planes start on 16 KiB boundaries, and k6/k3 read 10-30% differently at the heap's random phase. The reference binary needs the GGML_BENCH_THREADS define (harness README). # # Usage: kernel_ladder.sh [fmt,fmt,...] (default: every format) # KL_DASLANG the daslang binary (default: this tree's bin/daslang, or bin/Release/daslang.exe on @@ -11,11 +12,14 @@ # LCPP_TBO test-backend-ops (default: $HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe) # NTOK prefill tokens (default 512; 0 = decode rows only) # ROUNDS interleaved rounds per row (default 5) -# TEAM lanes: the bench dispatches the GEMV the engine's way (--team, DAS_JOBQUE_THREADS=N) -# and the reference runs GGML_BENCH_THREADS=N; unset = one thread, one raw call -# BIG=1 the decode row at d=32768 rows (a weight above any L3 - the many-lane comparison that +# TEAM lanes for the decode rows: the bench dispatches the GEMV the engine's way (--team, +# DAS_JOBQUE_THREADS=N) and the reference runs GGML_BENCH_THREADS=N. Default: the box's +# hardware threads (nproc). SOLO=1 = one thread, one raw call - the kernel table the +# tuner's races correspond to. +# BIG decode rows at d=32768 (a weight above any L3 - the many-lane comparison that # test-backend-ops perf's repeated 4096-row op does not give; the reference needs the -# m=32768 perf case, a two-line edit beside the thread define - HOW_TO_GET_SIDECAR.md) +# m=32768 perf case, a two-line edit beside the thread define - HOW_TO_GET_SIDECAR.md). +# Default 1 with TEAM, 0 with SOLO=1. # Output: a TSV table on stdout - fmt tier perm ours_us ours_med_us ref_us ratio - plus the box line. set -e -o pipefail # a failing daslang must not hide behind the join ROOT=$(cd "$(dirname "$0")/../../.." && pwd) # the tree this script lives in, whatever the env says @@ -27,8 +31,8 @@ TBO=${LCPP_TBO:-$HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-op FMTS=${1:-all} NTOK=${NTOK:-512} ROUNDS=${ROUNDS:-5} -TEAM=${TEAM:-0} -BIG=${BIG:-0} +SOLO=${SOLO:-0} +if [ "$SOLO" -gt 0 ]; then TEAM=0; BIG=${BIG:-0}; else TEAM=${TEAM:-$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)}; BIG=${BIG:-1}; fi ROWS=$(( BIG > 0 ? 32768 : 4096 )) THREADS=$(( TEAM > 0 ? TEAM : 1 )) TEAM_ARGS=""; [ "$TEAM" -gt 0 ] && TEAM_ARGS="--team" @@ -52,7 +56,7 @@ echo "# kernel ladder $(date +%F) box=$(hostname) daslang=$DASLANG ref=$TBO grep -q GGML_BENCH_THREADS "$TBO" || { echo "kernel_ladder: '$TBO' lacks the GGML_BENCH_THREADS define - it would run every core and the ratio column would lie (harness README)" >&2; exit 1; } set +e DAS_TUNE_MODE=normal DAS_JOBQUE_THREADS=$THREADS "$DASLANG" -jit "$ROOT/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das" \ - -- --fmt "$FMTS" --d "$ROWS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv $TEAM_ARGS > "$WORK/ours.raw" 2> "$WORK/ours.err" + -- --fmt "$FMTS" --d "$ROWS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv --base-align 4096 $TEAM_ARGS > "$WORK/ours.raw" 2> "$WORK/ours.err" rc=$? set -e awk -F'\t' 'NF >= 11' "$WORK/ours.raw" > "$WORK/ours.tsv" || true # the TSV rows; engine and tune notes fall away From fedd15bda313e84c6018f054e879231cf1c1bcd7 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:31:01 -0700 Subject: [PATCH 042/123] kernel parity plan: zen2 closing tables (v4 one-thread + engine shape) and zen4 v2 round one --- plans/kernel_parity_pass.md | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index d802138049..f0f8fb1b0f 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -201,6 +201,66 @@ above parity - the M1 CPU is closed for this pass. Decode floor k5 1.06x; the grid formats 1.16-1.51x (from 0.51-0.93x before the row-pair decode). Tiles 1.92-15.2x. +### zen2 v4 - the closing tables (2026-09-01, after the k6/k3 transposes and the fused flush) + +One thread at the HEAP phase (the ladder's engine-phase default came after this run; k6/k3 read 10-30% +better at the engine phase - k6 3527-3851, k3 2488-2893) and the engine shape (16 lanes, d=32768, +DRAM-bound; q8/mx4/k4 from the q8-family arm's validation run). + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | engine 16 lanes ours / ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 3348 / 4260 | 1.27 | 565887 / 890979 | 1.57 | 8906 / 9248 | 1.04 | +| k4 | 2337 / 2528 | 1.08 | 518533 / 839960 | 1.62 | 4472 / 4425 | 0.99 | +| k5 | 3553 / 3327 | 0.94 | 540857 / 1325657 | 2.45 | 5400 / 5475 | 1.01 | +| k6 | 3460 / 3652 | 1.06 | 630854 / 1083547 | 1.72 | 6477 / 6402 | 0.99 | +| q40 | 1973 / 2868 | 1.45 | 387611 / 1003869 | 2.59 | 4275 / 4575 | 1.07 | +| q51 | 2866 / 4118 | 1.44 | 656416 / 1844397 | 2.81 | 5842 / 6235 | 1.07 | +| iq4xs | 2097 / 3349 | 1.60 | 675053 / 1674064 | 2.48 | 4577 / 4119 | 0.90 | +| k3 | 3080 / 2601 | 0.84 | 678487 / 1255396 | 1.85 | 3662 / 3605 | 0.98 | +| iq3s | 7603 / 10242 | 1.35 | 660738 / 5179767 | 7.84 | 3934 / 5300 | 1.35 | +| iq3xxs | 6650 / 6573 | 0.99 | 659839 / 3308544 | 5.01 | 3854 / 3509 | 0.91 | +| iq4nl | 2117 / 3078 | 1.45 | 589114 / 1090653 | 1.85 | 4542 / 4533 | 1.00 | +| k2 | 1261 / 2083 | 1.65 | 658936 / 779599 | 1.18 | 2555 / 2711 | 1.06 | +| iq2s | 5058 / 5022 | 0.99 | 721269 / 2510375 | 3.48 | 3070 / 2887 | 0.94 | +| iq2xs | 4645 / 5354 | 1.15 | 735654 / 2706628 | 3.68 | 2749 / 2886 | 1.05 | +| iq2xxs | 5192 / 5103 | 0.98 | 655125 / 2526484 | 3.86 | 2994 / 2751 | 0.92 | +| mx4 | 2193 / 3004 | 1.37 | 621096 / 1433680 | 2.31 | 4143 / 4364 | 1.05 | + +Engine shape: every format 0.90-1.35 - q8 1.04, mx4 1.05, k4 0.99, k5 1.01, k6 0.99, q40 1.07, q51 1.07, +k3 0.98, iq3s 1.35, iq4nl 1.00, k2 1.06, iq2xs 1.05, iq2s 0.94, iq2xxs 0.92, iq3xxs 0.91, iq4xs 0.90. zen2 +is closed for this pass: k5 0.94 at one thread is the one row under the bar, and it is 1.01 at the engine shape. + +### zen4 v2 - after the transposes and the fused flush (2026-09-01, c7a.4xlarge EPYC 9R14, class x86-vnni512) + +TEST 90/90. One thread at the heap phase (the bootstrap ran before the ladder's engine-phase default) and the +engine shape (16 lanes, d=32768, DRAM-bound). The q8/mx4 engine rows here predate the q8 family's team arm +(one raw thread against 16) - round two re-measures them. + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | engine 16 lanes ours / ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 1614 / 3409 | 2.11 | 167078 / 500478 | 3.00 | 15344 / 6101 | 0.40 | +| k4 | 805 / 1865 | 2.32 | 328569 / 753160 | 2.29 | 3954 / 3028 | 0.77 | +| k5 | 2126 / 2615 | 1.23 | 178594 / 1120210 | 6.27 | 3788 / 3802 | 1.00 | +| k6 | 3045 / 2872 | 0.94 | 226619 / 782510 | 3.45 | 4840 / 4609 | 0.95 | +| q40 | 653 / 2743 | 4.20 | 172935 / 588322 | 3.40 | 3028 / 2963 | 0.98 | +| q51 | 1864 / 4607 | 2.47 | 239601 / 1431282 | 5.97 | 4149 / 4165 | 1.00 | +| iq4xs | 1180 / 2418 | 2.05 | 286066 / 1153787 | 4.03 | 2809 / 2848 | 1.01 | +| k3 | 833 / 2031 | 2.44 | 362356 / 978906 | 2.70 | 2318 / 2207 | 0.95 | +| iq3s | 7527 / 6087 | 0.81 | 259660 / 3089555 | 11.90 | 3903 / 3226 | 0.83 | +| iq3xxs | 5953 / 5234 | 0.88 | 260765 / 2664217 | 10.22 | 3122 / 2839 | 0.91 | +| iq4nl | 1066 / 3103 | 2.91 | 283855 / 625809 | 2.20 | 3084 / 3026 | 0.98 | +| k2 | 630 / 1276 | 2.03 | 429374 / 641108 | 1.49 | 1541 / 1605 | 1.04 | +| iq2s | 4917 / 3748 | 0.76 | 283651 / 1903473 | 6.71 | 2778 / 2116 | 0.76 | +| iq2xs | 4549 / 3616 | 0.80 | 283584 / 1844309 | 6.50 | 2312 / 2001 | 0.87 | +| iq2xxs | 4834 / 3915 | 0.81 | 260122 / 1991565 | 7.66 | 2464 / 2238 | 0.91 | +| mx4 | 1049 / 2740 | 2.61 | 293620 / 1348460 | 4.59 | 9599 / 2849 | 0.30 | + +One thread: every non-grid decode 0.94-4.20x (k6 0.94), tiles 1.5-11.9x; the grid formats 0.76-0.88x - +the x86 tail, unchanged. Engine shape: k5/q51/iq4xs/k2 1.00-1.04, q40/iq4nl 0.98, k6/k3 0.95, iq3xxs/iq2xxs +0.91, iq2xs 0.87, iq3s 0.83, iq2s 0.76 - and k4 0.77 (67 GB/s where the reference and our k5 pull 86; 2.32x +at one thread). Round two on the same box: SOLO at the engine phase, the corrected team rows, a tune-mode race +of k4/k6/iq2s through the team arm, perf stat on iq2s. + ## 3. Research memos (read before touching the kernels) - `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, From 6cb455c6c551163b78fff33383c6b0b12cb9289d Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:36:52 -0700 Subject: [PATCH 043/123] kernel parity plan: the M1 closing tables (one thread at the engine phase, and the 8-lane engine shape) --- plans/kernel_parity_pass.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index f0f8fb1b0f..d2909a1c8a 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -201,6 +201,30 @@ above parity - the M1 CPU is closed for this pass. Decode floor k5 1.06x; the grid formats 1.16-1.51x (from 0.51-0.93x before the row-pair decode). Tiles 1.92-15.2x. +### M1 v4 - closing tables (2026-09-01, after the k6/k3 transposes; one thread at the engine phase, and the engine shape: 8 lanes, d=32768) + +| fmt | decode ours / ref us | ratio | engine 8 lanes ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 1059 / 1537 | 1.45 | 4187 / 4519 | 1.08 | +| k4 | 840 / 1343 | 1.60 | 2240 / 2271 | 1.01 | +| k5 | 2068 / 2184 | 1.06 | 2757 / 2715 | 0.98 | +| k6 | 1699 / 2405 | 1.42 | 3224 / 3284 | 1.02 | +| q40 | 768 / 1623 | 2.11 | 2176 / 2231 | 1.03 | +| q51 | 2075 / 2752 | 1.33 | 2922 / 3145 | 1.08 | +| iq4xs | 987 / 1699 | 1.72 | 2234 / 2125 | 0.95 | +| k3 | 1876 / 2621 | 1.40 | 2016 / 2831 | 1.40 | +| iq3s | 4924 / 5714 | 1.16 | 5282 / 6096 | 1.15 | +| iq3xxs | 3813 / 4876 | 1.28 | 3949 / 5208 | 1.32 | +| iq4nl | 1036 / 1828 | 1.76 | 2175 / 2272 | 1.04 | +| k2 | 1390 / 1918 | 1.38 | 1463 / 2138 | 1.46 | +| iq2s | 3255 / 4932 | 1.52 | 3669 / 5366 | 1.46 | +| iq2xs | 2637 / 3261 | 1.24 | 2835 / 3490 | 1.23 | +| iq2xxs | 2913 / 3440 | 1.18 | 3112 / 3718 | 1.19 | +| mx4 | 1272 / 1910 | 1.50 | 1990 / 2147 | 1.08 | + +One thread: every decode >= 1.06 (k5 the floor; k6 1.42, k3 1.40). Engine shape: every format >= 0.95 (iq4xs), +most 1.0-1.5. The M1 CPU is closed at both shapes. + ### zen2 v4 - the closing tables (2026-09-01, after the k6/k3 transposes and the fused flush) One thread at the HEAP phase (the ladder's engine-phase default came after this run; k6/k3 read 10-30% From 0455b1129ea6ac41f7ee9877593e8f9e76f8e344 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:45:04 -0700 Subject: [PATCH 044/123] grid decode: DASLLAMA_GRID_ROWS_X86=1 re-arms the x86 row-group compose as a lab knob; the sidecar how-to learns the macOS box zen2 -> zen4 sped every kernel up 1.5-3x except the grid decodes (iq2xs 4645 -> 4562 us), which is the signature of a latency chain (index -> grid load -> panel store -> reload), not of vector throughput - so the register compose zen2 killed may win on zen4. The env knob (read at emit time; clear .jitted_scripts between A/B runs, the JIT cache does not key on it) puts the maddubs / vpdpbusd / vpdpbssd arms back behind grid_rows_path for that experiment; default off, the emission byte-identical to before. TEST 90/90 with the knob off and on. The how-to gains the M4 Pro walk: user-local cmake/ninja where Homebrew is another user's, -DDAS_FLEX_BISON_DISABLED=ON against Apple's bison 2.3, -DDAS_HV_DISABLED=ON without OpenSSL, TEAM = the performance cores. The plan carries zen4 round two and the M1 closing tables. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/HOW_TO_GET_SIDECAR.md | 10 ++++++ .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 35 ++++++++++++++++--- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- plans/kernel_parity_pass.md | 35 +++++++++++++++++++ 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md index 7c3930f47e..5f12422391 100644 --- a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -64,6 +64,16 @@ ls -la bin/daslang lib/LLVM.dll downloads the prebuilt `lib/LLVM.dll` for the platform (that name on every OS) - if `ls` does not show it, nothing below runs. Only `daslang` needs building: dasLLAMA is das, its native needs are builtins. +### A macOS box you do not own (the M4 Pro walk) + +Homebrew there may belong to another user (`/opt/homebrew` not writable) - do not chown it. CMake and +Ninja come as user-local binaries instead: the Kitware `cmake--macos-universal.tar.gz` and the +ninja-build `ninja-mac.zip` unpacked under `~/tools`, prepended to `PATH`. Apple's `/usr/bin/bison` is +2.3 and cannot read `ds2_parser.ypp`, so configure with `-DDAS_FLEX_BISON_DISABLED=ON` (the committed +generated parser is current); no OpenSSL means `-DDAS_HV_DISABLED=ON`. The reference builds with +`-DGGML_METAL=OFF -DGGML_BLAS=OFF` (CPU only). On a P+E chip give the ladder the performance cores +(`TEAM=10` on the M4 Pro), not `nproc`. + ## 3. The reference exe The kernel ladder's other side. Pin the same tip the dev box's tables use (`6c84c7d5d` for the diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 44f92394fe..4a7006b730 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -11,6 +11,7 @@ require llvm/daslib/llvm_jit_code require llvm/daslib/llvm_jit_common require daslib/ast_boost require dasllama/dasllama_gemm_schema +require daslib/fio // The perm-parameterized Q8 GEMM tile generator (gemm_generator_plan.md part 3, promoted from // the M2 hand perm): emits the mr-row x 4-token register tile as LLVM IR — the loop nest, @@ -1444,8 +1445,13 @@ def private sign_col_pm1(var te : TileEmit; var col : LLVMOpaqueValue?; r0, l : } //! the sdot lattice's grid decode composes row groups straight from the grid words; x86 keeps the panel (the -//! insert form measured 1.2-1.5x slower on zen2 - the ledger, followup 61) -def private grid_rows_path(te : TileEmit) : bool => te.dotKind == DOT_SDOT && te.rv == 4 +//! insert form measured 1.2-1.5x slower on zen2 - the ledger, followup 61) unless DASLLAMA_GRID_ROWS_X86=1, +//! the lab knob for boxes where the panel's latency chain, not the vector width, bounds the decode +def private grid_rows_path(te : TileEmit) : bool { + if (te.dotKind == DOT_SDOT) return te.rv == 4 + if (te.dotKind != DOT_MADDUBS && te.dotKind != DOT_VPDPBUSD && te.dotKind != DOT_VPDPBSSD) return false + return get_env_variable("DASLLAMA_GRID_ROWS_X86") == "1" +} //! eight +-1 bytes for sign byte `sb`, lane k negative when bit k is set def private pm1_of(sb : int) : uint64 { @@ -1687,11 +1693,30 @@ def private concat_rows(var te : TileEmit; var rows : array; l return concat_rows(te, nxt, 2 * lanes) } -//! one row group's sdot with its +-1 sign bytes multiplied into the magnitudes +//! one row group's dot with its +-1 sign bytes: the signed-weight lattices (sdot, vpdpbssd) multiply them +//! into the magnitudes; the unsigned-weight ones (maddubs, vpdpbusd) put them on the activation copy def private grid_rows_dot(var te : TileEmit; var acc, w, sg, xv : LLVMOpaqueValue?) : LLVMOpaqueValue? { let b = te.builder - var args <- [acc, LLVMBuildMul(b, w, sg, "ws"), xv] - return LLVMBuildCall2(b, te.sdot_ty, te.sdot_decl, args, "dot") + if (te.dotKind == DOT_SDOT || te.dotKind == DOT_VPDPBSSD) { + var args <- [acc, LLVMBuildMul(b, w, sg, "ws"), xv] + return LLVMBuildCall2(b, te.dotKind == DOT_SDOT ? te.sdot_ty : te.dp_ty, te.dotKind == DOT_SDOT ? te.sdot_decl : te.dp_decl, args, "dot") + } + var xs : LLVMOpaqueValue? + if (te.width == 256) { + var pargs <- [xv, sg] + xs = LLVMBuildCall2(b, te.psign_ty, te.psign_decl, pargs, "signx") + } else { + var neg = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, sg, LLVMConstNull(te.vwi8), "sneg") + xs = LLVMBuildSelect(b, neg, LLVMBuildSub(b, LLVMConstNull(te.vwi8), xv, ""), xv, "signx") + } + if (te.dotKind == DOT_VPDPBUSD) { + var dargs <- [acc, w, xs] + return LLVMBuildCall2(b, te.dp_ty, te.dp_decl, dargs, "dot") + } + var margs <- [w, xs] + var pairs = LLVMBuildCall2(b, te.madd_ty, te.madd_decl, margs, "pairs") + var wargs <- [pairs, ones_i16(te)] + return LLVMBuildAdd(b, acc, LLVMBuildCall2(b, te.maddwd_ty, te.maddwd_decl, wargs, "quads"), "dot") } //! row `base`'s 8 raw grid weights for (blk, l) as <8 x i8>: the iq2 formats' u64 grid entry in one diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index a97c4bea63..ba20d1962b 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5eul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5ful // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index d2909a1c8a..a041f79056 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -285,6 +285,34 @@ the x86 tail, unchanged. Engine shape: k5/q51/iq4xs/k2 1.00-1.04, q40/iq4nl 0.98 at one thread). Round two on the same box: SOLO at the engine phase, the corrected team rows, a tune-mode race of k4/k6/iq2s through the team arm, perf stat on iq2s. +Round two (the branch at 6d75a0956): + +| fmt | one thread, engine phase ours / ref us | ratio | engine 16 lanes ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 1874 / 3657 | 1.95 | 6575 / 6058 | 0.92 | +| k4 | 833 / 1906 | 2.29 | 3067 / 3022 | 0.99 | +| k5 | 2046 / 2885 | 1.41 | 4043 / 3794 | 0.94 | +| k6 | 2866 / 2686 | 0.94 | 4861 / 4600 | 0.95 | +| q40 | 737 / 2794 | 3.79 | 3163 / 2996 | 0.95 | +| q51 | 1690 / 4630 | 2.74 | 3940 / 4172 | 1.06 | +| iq4xs | 1176 / 2438 | 2.07 | 3036 / 2834 | 0.93 | +| k3 | 837 / 2016 | 2.41 | 2268 / 2209 | 0.97 | +| iq3s | 7571 / 6104 | 0.81 | 4128 / 3213 | 0.78 | +| iq3xxs | 5973 / 5242 | 0.88 | 3113 / 2802 | 0.90 | +| iq4nl | 1092 / 3055 | 2.80 | 2915 / 3025 | 1.04 | +| k2 | 623 / 1289 | 2.07 | 1461 / 1596 | 1.09 | +| iq2s | 4918 / 3763 | 0.77 | 2525 / 2129 | 0.84 | +| iq2xs | 4562 / 3633 | 0.80 | 2319 / 2001 | 0.86 | +| iq2xxs | 4857 / 3920 | 0.81 | 2461 / 2127 | 0.86 | +| mx4 | 1090 / 2763 | 2.54 | 2787 / 2844 | 1.02 | + +Engine shape: q51 1.06, iq4nl 1.04, k2 1.09, mx4 1.02, k4 0.99 (the 0.77 above was a one-off), k3 0.97, k6 0.95, +q40 0.95, k5 0.94, iq4xs 0.93, q8 0.92 - and the grid formats iq3xxs 0.90, iq2xs/iq2xxs 0.86, iq2s 0.84, iq3s +0.78: compute-bound even at 16 lanes (iq3s pulls 49 GB/s of the box's 87). The tune race through the team arm +(d=32768, 16 lanes): k4 maddubs256 3084 / vpdpbusd256 3030 / vpdpbusd512 2969 and k6 4630 / 4611 / 4532 confirm +the crowns; iq2s 2528 / 2383 / 2530 - the crowned width-512 seat loses 6% to vpdpbusd256 at the engine shape, +the gemv inheriting the tile's crown (tune_companion). perf stat gave nothing (no PMU in the VM). + ## 3. Research memos (read before touching the kernels) - `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, @@ -447,6 +475,13 @@ stays open. at 1.00-1.36x there - it must not lose), then zen4 (rent) where the gap lives. 3. NEXT x86 zen4 grid 0.77-0.88x: rent the c7a again with `perf stat` (uops per port, store-forwarding stalls) on iq2xs/iq3xxs decode against the reference exe's same shape - one proven fact before any form. + 2026-09-01, the fact the two boxes give for free: zen2 -> zen4 sped every kernel up 1.5-3x (k4 2337 -> + 833 us, k6 3460 -> 2866) EXCEPT the grid decodes (iq2xs 4645 -> 4562, iq2s 5058 -> 4918, iq3s 7603 -> + 7571) - a wider, faster core buys them nothing, so they are bound by a latency chain (index -> grid + load -> panel store -> reload), not by vector throughput. That is the case for the register compose + on zen4 even though zen2 killed it: `DASLLAMA_GRID_ROWS_X86=1` re-arms the x86 row-group arms as a + lab knob (env, read at emit time - clear `.jitted_scripts` between A/B runs, the JIT cache does not + key on it). A/B on zen4 next; if it wins there and loses on zen2 the gate becomes the tune class. CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x. The memo (`kernel_parity_research_arm.md`): the sdot count is at parity (2 per 32 weights per row, both sides); From 6a6ee904017f971ab1d00e9558d9cc8d36f8b269 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:51:12 -0700 Subject: [PATCH 045/123] grid gather panel and the tile's byte-expand scratch: a cache line's alignment; the row form reads bytes, not column dwords The gather panel alloca carried a 32-byte alignment - half of every 64-byte load at width 512 straddled a cache line (none did at width 256, which is why zen2 never showed it) - and the k5/k6/grid tiles' byte-expand scratch was a plain das array at the allocator's 16: both now start on a 64-byte boundary. zen2 tiles unchanged within their bands (k5 498-500, k6 589-605, iq2s 786-793 ms); the payoff is the width-512 lattices. Under the DASLLAMA_GRID_ROWS_X86 knob the row-group decode reads bytes where the panel form reads column dwords: zen4 measured the row form at 1.8x slower on iq2xs / iq3xxs (the two dword-read formats) while iq2s went 4917 -> 3559-3656 us (reference 3757), iq3s 7577 -> 6267-6419 (6105), iq2xxs 4860 -> 4302-4485. TEST 90/90 with the knob on. LLVM_JIT_CODEGEN_VERSION 0x60. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/dasllama/dasllama_gemm_gen.das | 6 +++++- modules/dasLLAMA/dasllama/dasllama_math_gen.das | 5 ++++- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 4a7006b730..d368e2b0b9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -249,6 +249,7 @@ struct private TileEmit { iq3s_panel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) sign_code_pm1_g : LLVMOpaqueValue? // the row-group grid decode: ksigns code -> eight +-1 bytes, [128 x i64], parity bit folded in + gridRows : bool // set while the row-group grid decode emits: the column dword read (a panel-form win) reverts to byte loads gfn : LLVMOpaqueValue? // the function under emit (block creation for emitted inner loops) gctx : LLVMOpaqueContext? psign_ty : LLVMOpaqueType? @@ -1344,6 +1345,7 @@ def private wg_col(mr, pos : int) : int => (pos / 4) * mr * 4 + pos % 4 //! the byte at packed position `pos`, read as one dword load of its column with the byte split off in //! registers - the four positions of a column share the load through CSE def private wg_col_byte(var te : TileEmit; var base : LLVMOpaqueValue?; pos : int) : LLVMOpaqueValue? { + if (te.gridRows) return wg_byte(te, base, wg_col(te.interleave, pos)) // the row form measured 1.8x slower on the dword read (zen4 iq2xs / iq3xxs) let b = te.builder var col = wg_i32(te, base, (pos / 4) * te.interleave * 4) let sh = 8 * (pos % 4) @@ -1850,7 +1852,9 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var } } if (pairs) { + te.gridRows = true emit_grid_rows_blk(te, wbP, xb, blk, signCol, a, a1, iq2) + te.gridRows = false } for (j in range(pairs ? 0 : 4)) { for (qd in range(rq)) { @@ -3064,7 +3068,7 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { } sign_pm1_emit_globals(te, LLVMGetGlobalParent(gc.impl)) te.iq3s_panel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "iq3s.panel") - LLVMSetAlignment(te.iq3s_panel, 32u) + LLVMSetAlignment(te.iq3s_panel, 64u) // a cache line: at 32 half the width-512 panel loads straddled one } var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 9f05c22cd8..072d225b45 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -2072,10 +2072,13 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let packed = kq_reads_packed_planes(int64(fmt)) var scratch : array if (!packed) { - scratch |> resize(mr * nsb * 256l) + scratch |> resize(mr * nsb * 256l + 64l) // + a cache line: the panel starts 64-aligned, the tile's 32/64-byte loads never straddle one } unsafe { var scp = !packed ? addr(scratch[0]) : reinterpret(null) + if (!packed) { + scp += int64((64ul - (intptr(scp) & 63ul)) & 63ul) + } var tb = tb0 while (tb < tend) { let tbe = min(tb + TB, tend) diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index ba20d1962b..f219fe9ae9 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x5ful // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x60ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) From f54852166572a14bb67afb59d2b4e8fd497bac5c Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:55:11 -0700 Subject: [PATCH 046/123] grid decode, x86-vnni512: iq2s and iq2xxs take the row form by class; the knob forces either way On zen4 the register compose of the row form measured iq2s 4918 -> 3600 us (reference 3757) and iq2xxs 4866 -> 4300 at one thread, iq2xxs 2456 -> 2259 at 16 lanes; iq3s split (+17% alone, -10% at 16 lanes), iq2xs and iq3xxs lost 1.8x with either read shape, and zen2 loses on all five - so the form is a per-class, per-format choice: the x86-vnni512 class takes it for iq2s and iq2xxs, everything else keeps the panel. DASLLAMA_GRID_ROWS_X86=1 / 0 still forces the form for the lab. TEST 90/90 on zen2 (class x86-avx2, the panel). LLVM_JIT_CODEGEN_VERSION 0x61. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/dasllama/dasllama_gemm_gen.das | 15 +++++++++++---- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index d368e2b0b9..990ea3bd14 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -12,6 +12,7 @@ require llvm/daslib/llvm_jit_common require daslib/ast_boost require dasllama/dasllama_gemm_schema require daslib/fio +require llvm/daslib/llvm_tune // The perm-parameterized Q8 GEMM tile generator (gemm_generator_plan.md part 3, promoted from // the M2 hand perm): emits the mr-row x 4-token register tile as LLVM IR — the loop nest, @@ -1446,13 +1447,19 @@ def private sign_col_pm1(var te : TileEmit; var col : LLVMOpaqueValue?; r0, l : return LLVMBuildOr(b, m, splat_i8w(te, 1), name) } -//! the sdot lattice's grid decode composes row groups straight from the grid words; x86 keeps the panel (the -//! insert form measured 1.2-1.5x slower on zen2 - the ledger, followup 61) unless DASLLAMA_GRID_ROWS_X86=1, -//! the lab knob for boxes where the panel's latency chain, not the vector width, bounds the decode +//! the sdot lattice's grid decode composes row groups straight from the grid words. x86: the panel's latency +//! chain (index -> grid load -> panel store -> reload) does not scale with the core, so on the x86-vnni512 +//! class iq2s and iq2xxs take the row form (zen4: iq2s 4918 -> 3600 us, iq2xxs 4866 -> 4300, both shapes); +//! iq3s is mixed there (+17% alone, -10% at 16 lanes) and iq2xs / iq3xxs lose 1.8x, zen2 loses on all five - +//! those keep the panel. DASLLAMA_GRID_ROWS_X86=1 / 0 forces the form (a lab knob: clear .jitted_scripts +//! between runs, the JIT cache does not key on it) def private grid_rows_path(te : TileEmit) : bool { if (te.dotKind == DOT_SDOT) return te.rv == 4 if (te.dotKind != DOT_MADDUBS && te.dotKind != DOT_VPDPBUSD && te.dotKind != DOT_VPDPBSSD) return false - return get_env_variable("DASLLAMA_GRID_ROWS_X86") == "1" + let knob = get_env_variable("DASLLAMA_GRID_ROWS_X86") + if (knob == "1") return true + if (knob == "0") return false + return tune_cpu_class() == "x86-vnni512" && (te.kq == 23 || te.kq == 25) } //! eight +-1 bytes for sign byte `sb`, lane k negative when bit k is set diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index f219fe9ae9..174d2ef032 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x60ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x61ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) From df4409f75685d9b440e470ba7dbe9294a7acd74c Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:56:02 -0700 Subject: [PATCH 047/123] grid_rows_path: the comment at the three-line cap --- modules/dasLLAMA/dasllama/dasllama_gemm_gen.das | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 990ea3bd14..f5c51aea89 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1447,12 +1447,9 @@ def private sign_col_pm1(var te : TileEmit; var col : LLVMOpaqueValue?; r0, l : return LLVMBuildOr(b, m, splat_i8w(te, 1), name) } -//! the sdot lattice's grid decode composes row groups straight from the grid words. x86: the panel's latency -//! chain (index -> grid load -> panel store -> reload) does not scale with the core, so on the x86-vnni512 -//! class iq2s and iq2xxs take the row form (zen4: iq2s 4918 -> 3600 us, iq2xxs 4866 -> 4300, both shapes); -//! iq3s is mixed there (+17% alone, -10% at 16 lanes) and iq2xs / iq3xxs lose 1.8x, zen2 loses on all five - -//! those keep the panel. DASLLAMA_GRID_ROWS_X86=1 / 0 forces the form (a lab knob: clear .jitted_scripts -//! between runs, the JIT cache does not key on it) +//! the sdot lattice composes row groups straight from the grid words; on x86 the panel's latency chain does not +//! scale with the core, so x86-vnni512 takes the row form for iq2s and iq2xxs (zen4: 4918 -> 3600, 4866 -> 4300 us) +//! and the panel elsewhere (iq2xs / iq3xxs lose 1.8x, zen2 loses on all five); DASLLAMA_GRID_ROWS_X86=1 / 0 forces it (clear .jitted_scripts between runs) def private grid_rows_path(te : TileEmit) : bool { if (te.dotKind == DOT_SDOT) return te.rv == 4 if (te.dotKind != DOT_MADDUBS && te.dotKind != DOT_VPDPBUSD && te.dotKind != DOT_VPDPBSSD) return false From 04a122f4535e8a488a99720cf51901c1d87ca656 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:56:54 -0700 Subject: [PATCH 048/123] kernel parity plan: Intel v2 round one (engine shape), M4 v1 --- plans/kernel_parity_pass.md | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index a041f79056..aaea121c10 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -225,6 +225,63 @@ Decode floor k5 1.06x; the grid formats 1.16-1.51x (from 0.51-0.93x before the r One thread: every decode >= 1.06 (k5 the floor; k6 1.42, k3 1.40). Engine shape: every format >= 0.95 (iq4xs), most 1.0-1.5. The M1 CPU is closed at both shapes. +### Intel v2 - Granite Rapids (2026-09-01, c8i.4xlarge Xeon 6975P-C, class x86-amx, branch 095f4030c; TEST 90/90) + +The bootstrap's two ladders both ran the new default (engine shape: 16 lanes, d=32768) - the one-thread table and +the knob A/B run in round two. + +| fmt | engine 16 lanes ours / ref us | ratio | +|---|---|---| +| q8 | 4539 / 4835 | 1.07 | +| k4 | 2332 / 2849 | 1.22 | +| k5 | 3073 / 3862 | 1.26 | +| k6 | 5230 / 3587 | 0.69 | +| q40 | 2345 / 4923 | 2.10 | +| q51 | 3196 / 5238 | 1.64 | +| iq4xs | 2367 / 4355 | 1.84 | +| k3 | 2703 / 2579 | 0.95 | +| iq3s | 7310 / 7575 | 1.04 | +| iq3xxs | 5396 / 6121 | 1.13 | +| iq4nl | 2359 / 5928 | 2.51 | +| k2 | 1229 / 1598 | 1.30 | +| iq2s | 4283 / 3786 | 0.88 | +| iq2xs | 3962 / 4579 | 1.16 | +| iq2xxs | 3805 / 3853 | 1.01 | +| mx4 | 2467 / 5802 | 2.35 | + +The reference does not reach this box's DRAM on most formats (q4_0 4.9 ms for 264 MB = 54 GB/s) where ours does +(2.3 ms = 115 GB/s) - hence 2.1-2.5x on q40/iq4nl/mx4. Tails: k6 0.69 (5.2 ms against 3.5 - the reference +saturates DRAM on q6_K, our k6 stays compute-bound; the vpdpbusd512 seat has no i16 chain for the fused flush), +iq2s 0.88, k3 0.95. + +### M4 Pro v1 - first tables (2026-09-01, Apple M4 Pro 10P+4E, class arm-i8mm, on the arm-neon PROFILE through the chain - no arm-i8mm profile existed; TEST 90/90) + +One thread at the engine phase; the engine shape = 10 performance cores, d=32768. The SMMLA lattice's first real race is +the tile column. + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | engine 10 lanes ours / ref us | ratio | +|---|---|---|---|---|---|---| +| q8 | 693 / 1125 | 1.62 | 184237 / 425873 | 2.31 | 2179 / 2078 | 0.95 | +| k4 | 649 / 916 | 1.41 | 210798 / 458459 | 2.17 | 1107 / 1132 | 1.02 | +| k5 | 1625 / 1464 | 0.90 | 201095 / 766460 | 3.81 | 1556 / 1521 | 0.98 | +| k6 | 1304 / 1731 | 1.33 | 329965 / 732022 | 2.22 | 1611 / 1824 | 1.13 | +| q40 | 596 / 1262 | 2.12 | 218731 / 516770 | 2.36 | 1055 / 1342 | 1.27 | +| q51 | 1637 / 1996 | 1.22 | 356194 / 1035551 | 2.91 | 1445 / 2306 | 1.60 | +| iq4xs | 780 / 1308 | 1.68 | 208597 / 670258 | 3.21 | 1091 / 1269 | 1.16 | +| k3 | 1435 / 1824 | 1.27 | 368506 / 955375 | 2.59 | 1257 / 1864 | 1.48 | +| iq3s | 3904 / 4061 | 1.04 | 155531 / 2102599 | 13.52 | 3298 / 3841 | 1.16 | +| iq3xxs | 2661 / 3755 | 1.41 | 155576 / 1927299 | 12.39 | 2263 / 3539 | 1.56 | +| iq4nl | 788 / 1430 | 1.81 | 231408 / 734682 | 3.17 | 1047 / 1387 | 1.32 | +| k2 | 969 / 1395 | 1.44 | 279559 / 710512 | 2.54 | 854 / 1302 | 1.52 | +| iq2s | 2386 / 2975 | 1.25 | 183660 / 1504690 | 8.19 | 2233 / 2572 | 1.15 | +| iq2xs | 1786 / 2435 | 1.36 | 184119 / 1250945 | 6.79 | 1530 / 2240 | 1.46 | +| iq2xxs | 2226 / 2623 | 1.18 | 155669 / 1336029 | 8.58 | 1892 / 2328 | 1.23 | +| mx4 | 966 / 1322 | 1.37 | 263140 / 679457 | 2.58 | 982 / 1521 | 1.55 | + +One thread: every decode >= 1.04 except k5 0.90; tiles 2.2-13.5x. Engine shape: everything >= 0.95 (q8), k5 0.98, +the rest 1.02-1.60. The arm-i8mm mint (the 1B vehicle, --tune, export) runs next on this box and the tables re-run +on the minted profile. + ### zen2 v4 - the closing tables (2026-09-01, after the k6/k3 transposes and the fused flush) One thread at the HEAP phase (the ladder's engine-phase default came after this run; k6/k3 read 10-30% From 12243671230801ceabc07b5fe27c32b497b0a5fd Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 08:59:33 -0700 Subject: [PATCH 049/123] grid row form, x86: the sign column's mask for every format - the ksigns table stays the sdot lattice's On x86 the row form's per-row sign table cost four scalar loads and three inserts per group; the panel form's vector-synthesized sign column plus one mask is the cheaper deposit there (the reverse of the M1, where the table beat five NEON ops). The x86 arms now take the column for all five grid formats; the sdot lattice keeps the table for the ksigns three. TEST 90/90 with the knob on. LLVM_JIT_CODEGEN_VERSION 0x62. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/dasllama/dasllama_gemm_gen.das | 4 ++-- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index f5c51aea89..9c7a5423d1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1745,7 +1745,7 @@ def private row_weights8(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l def private emit_grid_rows_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var signCol : LLVMOpaqueValue? [2]; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { let b = te.builder let n = te.width / 64 - let colSigns = te.kq == 33 || te.kq == 23 + let colSigns = te.kq == 33 || te.kq == 23 || te.dotKind != DOT_SDOT // x86: the sign column's mask beats four table loads + three inserts per group; the sdot lattice keeps the table for the ksigns formats for (qd in range(te.rq)) { var acc : LLVMOpaqueValue? [2] // per row group; the lo strip (l < 2) when iq2 var acc1 : LLVMOpaqueValue? [2] // iq2: the hi strip (l >= 2) @@ -1850,7 +1850,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var a1[i] = LLVMConstNull(te.vni32) } var signCol : LLVMOpaqueValue? [2] // this block's sign bytes, rows qd*rv.. at stride 4 - the gemv negates per vector, the tile's panel arrives signed - if (gather && (!pairs || te.kq == 33 || te.kq == 23)) { + if (gather && (!pairs || te.kq == 33 || te.kq == 23 || te.dotKind != DOT_SDOT)) { for (qd in range(rq)) { signCol[qd] = kq_sign_bytes(te, wbP, blk, qd, "sgc{blk}_{qd}") } diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 174d2ef032..b0b506ca06 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x61ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x62ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) From 0ae7686fe87507d933ff0d6efdf06b90de606caf Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:01:28 -0700 Subject: [PATCH 050/123] kernel parity plan: the zen4 row-form results, the class gate, the iq2s 16-lane suspect --- plans/kernel_parity_pass.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index aaea121c10..b919899e68 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -539,6 +539,19 @@ stays open. on zen4 even though zen2 killed it: `DASLLAMA_GRID_ROWS_X86=1` re-arms the x86 row-group arms as a lab knob (env, read at emit time - clear `.jitted_scripts` between A/B runs, the JIT cache does not key on it). A/B on zen4 next; if it wins there and loses on zen2 the gate becomes the tune class. + MEASURED on zen4 (one thread, engine phase; panel -> row form): iq2s 4918 -> 3559-3656 (reference 3757: + PARITY), iq3s 7577 -> 6267-6419 (6105; but -10% at 16 lanes), iq2xxs 4860 -> 4302-4485 (3922); iq2xs + 4548 -> 8072 and iq3xxs 5973 -> 10744 LOSE 1.8x with either read shape (dword column or bytes). The + sign path splits the ISAs too: the per-row +-1 table (the M1's win) costs x86 four scalar loads and + three inserts per group, the vector-synthesized sign column plus one mask is cheaper there - with it + zen2's row form reads iq2xxs 4696 (panel 4998) while iq2s/iq3s/iq2xs stay panel there. LANDED + (c6e99f81a, f68534c06): `grid_rows_path` = the x86-vnni512 class takes the row form for iq2s and + iq2xxs, every other x86 case the panel; the knob forces either. zen4 default check: iq2s 3595-3714 + (1.01-1.05x), iq2xxs 4383-4497 (0.87-0.89), iq2xs 4568-4577 (panel). OPEN: iq2s at 16 lanes on the + c7a (8 cores x 2 SMT) reads a 5.9 ms wall with one slot at 8.8 ms, both forms - iq2s's 8 KB grid table + plus panel plus x, twice per core, against a 32 KB L1 is the suspect (8-lane run next); the zen4 grid + residue at the engine shape is iq3s 0.78, iq2xs 0.86, iq3xxs 0.90 - the panel form's compute, kernel + work on the vnni512 lattice (a session of its own). CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x. The memo (`kernel_parity_research_arm.md`): the sdot count is at parity (2 per 32 weights per row, both sides); From 4592153fdeba4e6a0101204d30877a99cf0aee13 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:03:32 -0700 Subject: [PATCH 051/123] kernel parity plan: zen4 round four (column-sign row form) --- plans/kernel_parity_pass.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index b919899e68..44f07e6c7a 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -552,6 +552,15 @@ stays open. plus panel plus x, twice per core, against a 32 KB L1 is the suspect (8-lane run next); the zen4 grid residue at the engine shape is iq3s 0.78, iq2xs 0.86, iq3xxs 0.90 - the panel form's compute, kernel work on the vnni512 lattice (a session of its own). + Round four, the row form with the COLUMN signs on zen4 (one thread, engine phase; reference in + parentheses): iq2s 3541-3578 (3757: 1.05x), iq2xxs 2737-2771 (3922: 1.42x, from 0.89 with the table), + iq3xxs 4234-4322 (5243: 1.22x, from 0.56 with the table; the panel 0.88), iq3s 6238-6309 (6105: 0.97), + iq2xs 5381-5398 (3624: 0.67 - the panel's 0.80 stays). At 16 lanes on this 8-core x 2 SMT VM: iq2xxs + 1670 (panel 2456), iq3xxs 3415 (panel 3123 - loses), iq3s 4590 (panel 4145 - loses), iq2xs 2742 (panel + 2320), iq2s the 8.7 ms-slot anomaly. The one-thread wins that lose at 16 lanes point at SMT sharing + (two lanes' instruction streams per core); the 8-lane run decides what the class gate should say for + iq3xxs and iq3s. Tiles with the 64-aligned panel and scratch on zen4: unchanged (k5 178623, k6 225455, + iq3s 258990, iq2s 283977, iq2xs 284079 us). CPU decode on ARM (the M1 ladder): iq2xs 0.51x, iq2xxs 0.57x, iq3xxs 0.72x, iq3s 0.93x. The memo (`kernel_parity_research_arm.md`): the sdot count is at parity (2 per 32 weights per row, both sides); From e7249de2b1dd95b7e2d3b5407072204e26b1ffbc Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:04:36 -0700 Subject: [PATCH 052/123] kernel parity plan: Intel round two one-thread table --- plans/kernel_parity_pass.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 44f07e6c7a..73556eea66 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -254,6 +254,32 @@ The reference does not reach this box's DRAM on most formats (q4_0 4.9 ms for 26 saturates DRAM on q6_K, our k6 stays compute-bound; the vpdpbusd512 seat has no i16 chain for the fused flush), iq2s 0.88, k3 0.95. +Round two, one thread at the engine phase (branch eb94f1d9a): + +| fmt | decode ours / ref us | ratio | tile ours / ref us | ratio | +|---|---|---|---|---| +| q8 | 2224 / 2687 | 1.21 | 145999 / 578161 | 3.96 | +| k4 | 1140 / 1748 | 1.53 | 263222 / 787687 | 2.99 | +| k5 | 1525 / 2447 | 1.60 | 175535 / 1133333 | 6.46 | +| k6 | 1840 / 2040 | 1.11 | 239252 / 834122 | 3.49 | +| q40 | 1108 / 2288 | 2.06 | 175513 / 651009 | 3.71 | +| q51 | 1488 / 2946 | 1.98 | 214319 / 1410780 | 6.58 | +| iq4xs | 1126 / 2380 | 2.11 | 272987 / 1144376 | 4.19 | +| k3 | 1277 / 2266 | 1.77 | 331164 / 1115864 | 3.37 | +| iq3s | 5207 / 6783 | 1.30 | 252732 / 3499939 | 13.85 | +| iq3xxs | 4505 / 5053 | 1.12 | 252332 / 2480068 | 9.83 | +| iq4nl | 1122 / 2686 | 2.39 | 263188 / 717486 | 2.73 | +| k2 | 716 / 1468 | 2.05 | 356125 / 688367 | 1.93 | +| iq2s | 3996 / 3709 | 0.93 | 272607 / 1851295 | 6.79 | +| iq2xs | 3955 / 4342 | 1.10 | 272793 / 2184555 | 8.01 | +| iq2xxs | 3675 / 3760 | 1.02 | 252192 / 1799256 | 7.13 | +| mx4 | 1285 / 2990 | 2.33 | 273869 / 1247422 | 4.55 | + +One thread: every decode >= 1.02 except iq2s 0.93; k6 is 1.11 ALONE - its 0.69 at 16 lanes is a many-lane +effect on this 8-core x 2 SMT VM (two instruction streams per core), the same shape as zen4's iq3s/iq3xxs. +Tiles 1.9-13.9x. The knob A/B and the k6/k3/iq2s seat races follow in the same round. + + ### M4 Pro v1 - first tables (2026-09-01, Apple M4 Pro 10P+4E, class arm-i8mm, on the arm-neon PROFILE through the chain - no arm-i8mm profile existed; TEST 90/90) One thread at the engine phase; the engine shape = 10 performance cores, d=32768. The SMMLA lattice's first real race is From b5925fb27630d83a8fb3cdeae98aaf62b8ea156b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:06:41 -0700 Subject: [PATCH 053/123] kernel parity plan: the M4 minted-profile table --- plans/kernel_parity_pass.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 73556eea66..f23cf02ec9 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -307,6 +307,13 @@ the tile column. One thread: every decode >= 1.04 except k5 0.90; tiles 2.2-13.5x. Engine shape: everything >= 0.95 (q8), k5 0.98, the rest 1.02-1.60. The arm-i8mm mint (the 1B vehicle, --tune, export) runs next on this box and the tables re-run on the minted profile. +MINTED (fd8e490f1: provenance ok / ok / neon;dotprod;i8mm;bf16;fullfp16;lse, 49 kernels): the kq tile crowns are +the neon profile's (`mr8` everywhere), q8q8_tile_gen moves mr8_budget -> kstep2, seven [tuned] loop kernels pick +other widths. One thread on the minted profile, us (reference): q8 727 (1128) 1.55, k4 666 (916) 1.38, k5 1643 +(1466) 0.89, k6 1312 (1732) 1.32, q40 605 (1263) 2.09, q51 1651 (1994) 1.21, iq4xs 815 (1311) 1.61, k3 1454 (1829) +1.26, iq3s 3888 (4066) 1.05, iq3xxs 2678 (3762) 1.40, iq4nl 795 (1433) 1.80, k2 976 (1404) 1.44, iq2s 2383 (2952) +1.24, iq2xs 1787 (2441) 1.37, iq2xxs 2224 (2616) 1.18, mx4 995 (1322) 1.33 - the same table as before, as the +crowns say. k5 0.89 is the M4's one tail (the M1 has it at 1.06; the P-core's byte->lane deposit costs more). ### zen2 v4 - the closing tables (2026-09-01, after the k6/k3 transposes and the fused flush) From f2f890863c9343235dde8c388a7c82b768da8a70 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:08:32 -0700 Subject: [PATCH 054/123] kernel parity plan: Intel seat races (k6 at parity at 16 lanes), the gemv-own-crown item queued --- plans/kernel_parity_pass.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index f23cf02ec9..f30ef2135d 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -275,9 +275,16 @@ Round two, one thread at the engine phase (branch eb94f1d9a): | iq2xxs | 3675 / 3760 | 1.02 | 252192 / 1799256 | 7.13 | | mx4 | 1285 / 2990 | 2.33 | 273869 / 1247422 | 4.55 | -One thread: every decode >= 1.02 except iq2s 0.93; k6 is 1.11 ALONE - its 0.69 at 16 lanes is a many-lane -effect on this 8-core x 2 SMT VM (two instruction streams per core), the same shape as zen4's iq3s/iq3xxs. -Tiles 1.9-13.9x. The knob A/B and the k6/k3/iq2s seat races follow in the same round. +One thread: every decode >= 1.02 except iq2s 0.93; k6 1.11. Tiles 1.9-13.9x. +The seat races (tune mode, every registry row; us): k6 one thread maddubs256 1648 / vpdpbusd256 1636 / +vpdpbusd512 1615 (reference 1952), at 16 lanes d=32768 3489 / 3424 / 3419 against the reference's 3587 - +0.98-1.05x, so the bootstrap ladder's k6 0.69 was that first run's artifact, not the kernel (round three +re-measures it cold). k3: 1246 / 1125 / 922 alone (reference 2258), 1780 / 1797 / 1525 at 16 lanes +(2579). iq2s: 4060 / 3855 / 3957 alone (3702), 4242 / 4090 / 4208 at 16 lanes (3786) - the crowned width-512 +seat loses 3% to vpdpbusd256 at the engine shape, as on zen4: the gemv inherits the tile's crown +(tune_companion), and the tile and the gemv do not agree on width. The knob A/B (the table-sign row form): +iq2s 5345, iq2xs 5033, iq2xxs 3754-3820, iq3s 9763, iq3xxs 5606 against the panel's 3956 / 3902 / 3904 / +5276 / 4601 - the panel everywhere but iq2xxs (marginal); the column-sign form is round three's question. ### M4 Pro v1 - first tables (2026-09-01, Apple M4 Pro 10P+4E, class arm-i8mm, on the arm-neon PROFILE through the chain - no arm-i8mm profile existed; TEST 90/90) @@ -433,6 +440,11 @@ the gemv inheriting the tile's crown (tune_companion). perf stat gave nothing (n ## 4. Work queue (one item at a time; a row is done at >= 1.0x on rig 2's shape AND the vehicle) +QUEUED (from the three-box rounds of 2026-09-01): the gemv's own crown. The tuner races the TILE and the gemv +follows (`tune_companion`); on zen4 and Intel the gemv prefers `vpdpbusd_width256` where the tile crowns +width 512 (iq2s 3-6%, k4 similar) - the mint should race the gemv seats at n=1 in the same walk and stamp +them separately. A tune-framework change (llvm_tune + the seat declarations), a mint per class after it. + CPU decode (gap 2) - the k-quant decode kernels were the tails the full ladder exposed (k3 0.80x, k6 0.90x, k5 0.91x): 1. DONE k3 + k2: one i16 flush per sub-block (the per-16 chains are bounded at 4 x 1778 / 4 x 762) - From 32911d4fea37b6a59d1cc0644aa2405bfd429fcb Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:09:41 -0700 Subject: [PATCH 055/123] kernel parity plan: the M4 engine-shape table on the minted profile --- plans/kernel_parity_pass.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index f30ef2135d..80d2bee838 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -321,6 +321,12 @@ other widths. One thread on the minted profile, us (reference): q8 727 (1128) 1. 1.26, iq3s 3888 (4066) 1.05, iq3xxs 2678 (3762) 1.40, iq4nl 795 (1433) 1.80, k2 976 (1404) 1.44, iq2s 2383 (2952) 1.24, iq2xs 1787 (2441) 1.37, iq2xxs 2224 (2616) 1.18, mx4 995 (1322) 1.33 - the same table as before, as the crowns say. k5 0.89 is the M4's one tail (the M1 has it at 1.06; the P-core's byte->lane deposit costs more). +Engine shape on the minted profile (10 P-cores, d=32768), ours / reference us: q8 2178 / 2082 (0.96), k4 1116 / +1146 (1.03), k5 1549 / 1546 (1.00), k6 1627 / 1843 (1.13), q40 1057 / 1368 (1.29), q51 1447 / 2344 (1.62), iq4xs +1120 / 1292 (1.15), k3 1244 / 1909 (1.53), iq3s 3308 / 4059 (1.23), iq3xxs 2258 / 3735 (1.65), iq4nl 1055 / 1451 +(1.38), k2 850 / 1343 (1.58), iq2s 2242 / 2624 (1.17), iq2xs 1698 / 2343 (1.38), iq2xxs 1894 / 2416 (1.28), mx4 +983 / 1553 (1.58). The M4 CPU is at or above parity on every engine-shape row; k5 0.89 alone is the one tail, +and the NEON per-lane byte shift for its high-bit deposit (two ops per half instead of five) is measured next. ### zen2 v4 - the closing tables (2026-09-01, after the k6/k3 transposes and the fused flush) From 111becf5b0ed1aaeb7f4c12c9559473422b6e0e2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:11:25 -0700 Subject: [PATCH 056/123] kernel parity plan: the M4 k5 deposit result - the M4 closed --- plans/kernel_parity_pass.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 80d2bee838..1a138e13e1 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -325,8 +325,11 @@ Engine shape on the minted profile (10 P-cores, d=32768), ours / reference us: q 1146 (1.03), k5 1549 / 1546 (1.00), k6 1627 / 1843 (1.13), q40 1057 / 1368 (1.29), q51 1447 / 2344 (1.62), iq4xs 1120 / 1292 (1.15), k3 1244 / 1909 (1.53), iq3s 3308 / 4059 (1.23), iq3xxs 2258 / 3735 (1.65), iq4nl 1055 / 1451 (1.38), k2 850 / 1343 (1.58), iq2s 2242 / 2624 (1.17), iq2xs 1698 / 2343 (1.38), iq2xxs 1894 / 2416 (1.28), mx4 -983 / 1553 (1.58). The M4 CPU is at or above parity on every engine-shape row; k5 0.89 alone is the one tail, -and the NEON per-lane byte shift for its high-bit deposit (two ops per half instead of five) is measured next. +983 / 1553 (1.58). The M4 CPU is at or above parity on every engine-shape row; k5 0.89 alone was the one tail. +LANDED: the NEON per-lane byte shift for k5's high-bit deposit (`ushl` / `ushr` + `and` per half, where the mask +test took five ops; sdot lattice only - x86 has no byte shift): M4 k5 decode 1625-1643 -> 1357 us (reference +1466: 1.08x), engine shape 1549 -> 1360 (1.14x), tile unchanged, k4 unchanged; TEST 90/90 on the M4 and zen2. +The M4 is closed on every row at both shapes. ### zen2 v4 - the closing tables (2026-09-01, after the k6/k3 transposes and the fused flush) From 1e94e38c6f857a0bda2210039cf5999bf91d45fa Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:11:26 -0700 Subject: [PATCH 057/123] grid decode: the x86-amx class takes the row form for iq2xxs and iq3xxs; k5's high-bit deposit shifts bytes on NEON Granite Rapids (c8i.4xlarge, Xeon 6975P-C) with the column-sign row form: iq2xxs 3678 -> 2518 us alone (reference 3708) and 3811 -> 2614 at 16 lanes (3853), iq3xxs 5448 -> 4515 at 16 lanes (6121) and 4504 -> 4392 alone; iq2s, iq2xs and iq3s stay faster on the panel there. zen4 keeps iq2s + iq2xxs (its iq3xxs wins alone and loses at 16 SMT lanes). The k5 deposit on the sdot lattice: NEON shifts bytes per lane, so bit t (lo) / 4 + t (hi) moves onto bit 4 with one ushl / ushr and one and, where the mask test took five ops per half; x86 has no byte shift and keeps the mask form. TEST 90/90 on zen2. LLVM_JIT_CODEGEN_VERSION 0x63. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 21 ++++++++++++++----- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 9c7a5423d1..f07de416e6 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -945,9 +945,17 @@ def private splat_lane_i32(var te : TileEmit; var v : LLVMOpaqueValue?; lane : i } // or-in of a masked bit test as the nibble's 0x10: w |= (bytes & mask) != 0 ? 0x10 : 0 -// (pcmpeqb+pand shape after lowering — the k5 high-bit deposit) -def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { +// (pcmpeqb+pand shape after lowering — the k5 high-bit deposit). NEON shifts bytes per lane, so the sdot +// lattice moves bit t (lo) / 4 + t (hi) straight onto bit 4: one ushl / ushr and one and +def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; hi : bool; name : string) : LLVMOpaqueValue? { let b = te.builder + if (te.dotKind == DOT_SDOT) { + let w8 = te.width / 8 + var shifts <- [for (i in range(w8)); LLVMConstInt(te.types.t_int8, uint64(hi ? i % 4 : 4 - i % 4), 0)] + var shv = LLVMConstVector(array_data_ptr(shifts), uint(w8)) + var moved = hi ? LLVMBuildLShr(b, bytes, shv, "") : LLVMBuildShl(b, bytes, shv, "") + return LLVMBuildOr(b, w, LLVMBuildAnd(b, moved, splat_i8w(te, 16), ""), name) + } var hit = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, LLVMBuildAnd(b, bytes, maskv, ""), LLVMConstNull(te.vwi8), "") var sel = LLVMBuildSelect(b, hit, splat_i8w(te, 16), LLVMConstNull(te.vwi8), name) return LLVMBuildOr(b, w, sel, name) @@ -1098,8 +1106,8 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f if (te.kq == 5 && !te.kqBytes) { var hoff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(128 * mr + (blk * 4 + j) * mr + qd * te.rv)), "") var hb = load_row_bytes_x4(te, te.wg, hoff, "qh{blk}_{j * rq + qd}") - wlo = or_bit_x10(te, wlo, hb, maskLo, "wlo5{blk}_{j * rq + qd}") - whi = or_bit_x10(te, whi, hb, maskHi, "whi5{blk}_{j * rq + qd}") + wlo = or_bit_x10(te, wlo, hb, maskLo, false, "wlo5{blk}_{j * rq + qd}") + whi = or_bit_x10(te, whi, hb, maskHi, true, "whi5{blk}_{j * rq + qd}") } elif (k6 && !te.kqBytes) { // the plane's qh columns 2blk / 2blk + 1 hold this sub-block's four j sites at 2j: two loads per sub-block, nothing lives past it let hbase = 128 * mr + blk * 2 * 4 * mr @@ -1456,7 +1464,10 @@ def private grid_rows_path(te : TileEmit) : bool { let knob = get_env_variable("DASLLAMA_GRID_ROWS_X86") if (knob == "1") return true if (knob == "0") return false - return tune_cpu_class() == "x86-vnni512" && (te.kq == 23 || te.kq == 25) + let cls = tune_cpu_class() + if (cls == "x86-vnni512") return te.kq == 23 || te.kq == 25 // zen4: iq2s, iq2xxs (iq3xxs wins alone, loses at 16 SMT lanes) + if (cls == "x86-amx") return te.kq == 25 || te.kq == 34 // Granite Rapids: iq2xxs 1.47x, iq3xxs 1.36x at the engine shape + return false } //! eight +-1 bytes for sign byte `sb`, lane k negative when bit k is set diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index b0b506ca06..03e5e08565 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x62ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x63ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) From 3bd22a1270e6bc408623ce890d58144135f66994 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:14:31 -0700 Subject: [PATCH 058/123] kernel parity plan: Intel round three and the amx gate verification --- plans/kernel_parity_pass.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 1a138e13e1..921174ba92 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -285,6 +285,15 @@ seat loses 3% to vpdpbusd256 at the engine shape, as on zen4: the gemv inherits (tune_companion), and the tile and the gemv do not agree on width. The knob A/B (the table-sign row form): iq2s 5345, iq2xs 5033, iq2xxs 3754-3820, iq3s 9763, iq3xxs 5606 against the panel's 3956 / 3902 / 3904 / 5276 / 4601 - the panel everywhere but iq2xxs (marginal); the column-sign form is round three's question. +Round three (f68534c06, the column-sign row form), one thread panel -> rows: iq2xxs 3678 -> 2518 (reference 3708: +1.47x), iq3xxs 4504 -> 4392 (5092: 1.16x); iq2s 3994 -> 5428, iq2xs 3962 -> 5504, iq3s 5209 -> 9785 stay panel. +At 16 lanes: iq2xxs 3811 -> 2614 (3853: 1.47x), iq3xxs 5448 -> 4515 (6121: 1.36x), the other three worse. Eight +lanes (one per core) against 16: k6 6562 -> 3420-5219, k3 3720 -> 2723, iq2s 4431 -> 4287, iq3s 9278 -> 7339 - +more lanes help every format here, 16 stands. LANDED (d9928f985): the x86-amx class takes the row form for iq2xxs +and iq3xxs; the default verified on the box - TEST 90/90, iq2xxs 2473-2496 alone / 2606 at 16 lanes (1.49x / +1.48x), iq3xxs 4390 / 4531 (1.16x / 1.35x), iq2s 3965-3991 / 4280 (0.93 / 0.88), iq2xs 3927-3973 / 3926 (1.10 / +1.17), iq3s 5081-5182 / 7254 (1.30 / 1.05), k6 at 16 lanes 5316 in normal mode against 3419-3489 for the same +stamped seat in the tune-mode race - a mode difference, not the kernel (the back-to-back probe runs next). ### M4 Pro v1 - first tables (2026-09-01, Apple M4 Pro 10P+4E, class arm-i8mm, on the arm-neon PROFILE through the chain - no arm-i8mm profile existed; TEST 90/90) From eec815e4dec79d772a05b08a812cd60416ca1762 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:16:31 -0700 Subject: [PATCH 059/123] kernel parity plan: the Intel k6 mode probe (the AVX-512 license shape), zen4 at 8 lanes --- plans/kernel_parity_pass.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 921174ba92..d56756894b 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -294,6 +294,11 @@ and iq3xxs; the default verified on the box - TEST 90/90, iq2xxs 2473-2496 alone 1.48x), iq3xxs 4390 / 4531 (1.16x / 1.35x), iq2s 3965-3991 / 4280 (0.93 / 0.88), iq2xs 3927-3973 / 3926 (1.10 / 1.17), iq3s 5081-5182 / 7254 (1.30 / 1.05), k6 at 16 lanes 5316 in normal mode against 3419-3489 for the same stamped seat in the tune-mode race - a mode difference, not the kernel (the back-to-back probe runs next). +The probe: same 16 lanes, same 128 x 256-row split, same seat - 5270 us when its calls run back to back (normal +mode), 3284-3448 when interleaved with ~60 ms of scalar rows (tune mode), three times over. That is the shape of +Intel's AVX-512 frequency license (sustained 512-bit work drops the core clock; the reference's q6_K runs 256-bit); +the seat-alone runs (each vector seat back to back at 16 lanes) decide it, and the mint's tile race (bursty per +seat) may have crowned width 512 where the sustained gemv wants 256 - the gemv-own-crown item once more. ### M4 Pro v1 - first tables (2026-09-01, Apple M4 Pro 10P+4E, class arm-i8mm, on the arm-neon PROFILE through the chain - no arm-i8mm profile existed; TEST 90/90) @@ -615,6 +620,10 @@ stays open. plus panel plus x, twice per core, against a 32 KB L1 is the suspect (8-lane run next); the zen4 grid residue at the engine shape is iq3s 0.78, iq2xs 0.86, iq3xxs 0.90 - the panel form's compute, kernel work on the vnni512 lattice (a session of its own). + zen4 at 8 lanes (one per core, d=32768; reference at 8 threads): iq2s 6398 (4236: 0.66), iq2xxs 2852 + (4206: 1.47x), iq2xs 4582 (3975: 0.87), iq3s 7668 (6410: 0.84), k6 5013 (4878: 0.97, 77 GB/s = the + box's DRAM). iq2s scales at 56% of its one-thread rate (147 MB in 6.4 ms = 23 GB/s, not bandwidth) + where iq2xxs scales at 96% - the 8 KB grid table's L1 pressure is the suspect. Round four, the row form with the COLUMN signs on zen4 (one thread, engine phase; reference in parentheses): iq2s 3541-3578 (3757: 1.05x), iq2xxs 2737-2771 (3922: 1.42x, from 0.89 with the table), iq3xxs 4234-4322 (5243: 1.22x, from 0.56 with the table; the panel 0.88), iq3s 6238-6309 (6105: 0.97), From f51bb431ebbeb38ea5ecf4aecb709aeccb2740c2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:38:43 -0700 Subject: [PATCH 060/123] generated kernels: a 64-byte-aligned entry alloca realigns the frame - spill slots stop straddling cache lines The k-quant gemvs had no alloca, so their frames sat at whatever 16-byte phase the caller's RSP had; ymm/zmm spill slots then straddled cache lines or not by that phase. That was k6's per-process 4.0 / 5.4 ms draw on zen2 (stack ASLR, untouched by the arena's alignment) and the 5270-vs-3713 us gap between normal and tune mode for the same seat on Granite Rapids (two call paths, two RSP phases; Intel splits a line-straddling 64-byte access). One alloca [64 x i8] align 64 at every generated kernel's entry makes LLVM realign the frame. TEST 90/90. LLVM_JIT_CODEGEN_VERSION 0x64. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 20 +++++++++++++++++++ modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index f07de416e6..5801176524 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -961,6 +961,14 @@ def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; hi return LLVMBuildOr(b, w, sel, name) } +//! a 64-byte-aligned alloca at the kernel's entry: LLVM then realigns the frame, so every ymm/zmm spill slot sits +//! inside one cache line whatever RSP the caller arrives with (Intel splits a line-straddling 64-byte access; the +//! per-process stack phase was k6's 4.0 / 5.4 ms draw on zen2 and its normal-vs-tune mode gap on Granite Rapids) +def private frame_align64(b : LLVMOpaqueBuilder?; types : PrimitiveTypes?) { + var a = LLVMBuildAlloca(b, LLVMArrayType(types.t_int8, 64u), "frame64") + LLVMSetAlignment(a, 64u) +} + // One 256-weight SUPERBLOCK, K-quant grp form (te.kq set): weight vectors unpacked // ONCE per (sub-block, dword-group) and dotted against every token — weight-stationary, // bit-exact vs per-token GEMVs (tokCount=1). kq v2 folds integer sub-scales/bsums against Q8_K-form activations. @@ -2372,6 +2380,7 @@ def private emit_amx_tile(var gc : LlvmCodeCtx; p : TilePerm; f16s : bool) : boo var after = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "after") LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) var yp = LLVMGetParam(gc.impl, 0u) var wg = LLVMGetParam(gc.impl, 1u) var sg = LLVMGetParam(gc.impl, 2u) @@ -2519,6 +2528,7 @@ def private emit_amx_tile_pipelined(var gc : LlvmCodeCtx; p : TilePerm; f16s : b var after = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "after") LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) var yp = LLVMGetParam(gc.impl, 0u) var wg = LLVMGetParam(gc.impl, 1u) var sg = LLVMGetParam(gc.impl, 2u) @@ -2672,6 +2682,7 @@ def private tile_gen_impl(var gc : LlvmCodeCtx; f16s : bool) : bool { // entry: per-token activation/scale bases; nb = n / 32 LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = p.kstep) sa.yp = LLVMGetParam(gc.impl, 0u) te.wg = LLVMGetParam(gc.impl, 1u) @@ -2783,6 +2794,7 @@ def private witness_gen(var gc : LlvmCodeCtx) : bool { var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") let b = gc.jit.builder LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) if (dot_kind(p.dotPrim) == DOT_AMX) { // TMUL executes only after the per-process XTILEDATA grant, so the arch_prctl syscall IS the witness let mod = LLVMGetGlobalParent(gc.impl) @@ -2817,6 +2829,7 @@ def private cfg_gen(var gc : LlvmCodeCtx) : bool { var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") let b = gc.jit.builder LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) if (dot_kind(p.dotPrim) == DOT_AMX && p.latch != 0) { let mod = LLVMGetGlobalParent(gc.impl) var glob = LLVMGetNamedGlobal(mod, "dasllama.amx.tilecfg") @@ -2880,6 +2893,7 @@ def private gemv_gen_impl(var gc : LlvmCodeCtx; f16s : bool) : bool { // entry: group range off the row range; d=0/t0=0 turn emit_slice's store offset into the absolute row g*mr LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = p.gkstep) sa.yp = LLVMGetParam(gc.impl, 0u) let wp = LLVMGetParam(gc.impl, 1u) @@ -2947,6 +2961,7 @@ def private mx4_gemv_gen(var gc : LlvmCodeCtx) : bool { // entry: group range off the row range; plane strides are nibble n/2 and exponent n/32 LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = p.gkstep) sa.yp = LLVMGetParam(gc.impl, 0u) let wn = LLVMGetParam(gc.impl, 1u) @@ -3007,6 +3022,7 @@ def private mx4_tile_gen(var gc : LlvmCodeCtx) : bool { // entry: per-token activation/scale bases; wn/we arrive pre-offset to the group LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = p.kstep) sa.yp = LLVMGetParam(gc.impl, 0u) te.wg = LLVMGetParam(gc.impl, 1u) @@ -3066,6 +3082,7 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: group range off the row range; strides are disk footprints (quants 128/160/192B, scales 20/20/18B) LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) if (fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24 || fmt == 25) { // iq3s/iq3xxs/iq2s/iq2xs: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) te.gfn = gc.impl @@ -3153,6 +3170,7 @@ def private q51_gemv_gen(var gc : LlvmCodeCtx) : bool { var gexit = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "gexit") LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) let qgp = LLVMGetParam(gc.impl, 1u) @@ -3215,6 +3233,7 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: per-token activation/scale/bsum bases; the block unit is a superblock (nb = n/256) LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) te.wg = LLVMGetParam(gc.impl, 1u) @@ -3308,6 +3327,7 @@ def private q51_tile_gen(var gc : LlvmCodeCtx) : bool { // entry: per-token activation/scale/asum bases; the block unit is 32 weights (nb = n/32) LLVMPositionBuilderAtEnd(b, entry) + frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) te.wg = LLVMGetParam(gc.impl, 1u) diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 03e5e08565..17ab94c3f0 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x63ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x64ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) From 293180c582d2745640a4e3a4519bac1fcfe9feb9 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:46:30 -0700 Subject: [PATCH 061/123] kernel parity plan: the frame realignment, the first-format bench artifact, the Intel mode gap left open --- plans/kernel_parity_pass.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index d56756894b..4c92619fcd 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -299,6 +299,14 @@ mode), 3284-3448 when interleaved with ~60 ms of scalar rows (tune mode), three Intel's AVX-512 frequency license (sustained 512-bit work drops the core clock; the reference's q6_K runs 256-bit); the seat-alone runs (each vector seat back to back at 16 lanes) decide it, and the mint's tile race (bursty per seat) may have crowned width 512 where the sustained gemv wants 256 - the gemv-own-crown item once more. +The seat-alone runs said no to the license: 512 alone 3713, 256 alone 3551, maddubs alone 3541 (reference 3441) - +back to back, all near parity - so it is the MODE: the same seat generated under the adopted profile (normal mode) +runs 5159-5270 at 16 lanes and 1861-2244 / 1939-3622 (bimodal within a process) at one thread, where the tune-mode +race of the same name reads 3284-3713 and 1615-1789 / 1648-2059. zen2 shows no such gap (normal 3446-4033, tune +3657-4076). The 64-byte frame realignment (e0b3892b7) did not move it. The JIT'd body is emitted at runtime and is +not in the cache, so the two bodies could not be diffed here. OPEN, on the Intel box: whether the adopted perm +parses to the grid row's fields (defaults) or the runtime differs; the gemv's own in-situ race (queued) measures +every seat the same way and stamps what is fastest, which closes it either way. ### M4 Pro v1 - first tables (2026-09-01, Apple M4 Pro 10P+4E, class arm-i8mm, on the arm-neon PROFILE through the chain - no arm-i8mm profile existed; TEST 90/90) @@ -624,6 +632,17 @@ stays open. (4206: 1.47x), iq2xs 4582 (3975: 0.87), iq3s 7668 (6410: 0.84), k6 5013 (4878: 0.97, 77 GB/s = the box's DRAM). iq2s scales at 56% of its one-thread rate (147 MB in 6.4 ms = 23 GB/s, not bandwidth) where iq2xxs scales at 96% - the 8 KB grid table's L1 pressure is the suspect. + RESOLVED (the order test): the 8-9 ms slot belongs to whichever format is FIRST in a bench process - iq2s + first 4887 (slot 786-8747), iq2xxs first 1924 / 4063 (342-7920); the same format second is flat (iq2s + 3257 at 2861-3377). The team arm now burns in (six warmup dispatches per row). With iq2s second, its row + form at 16 lanes reads 3257 against the panel's 2525 (the ladder's row; reference 2159): the one-thread + win does not survive the engine shape, so iq2s left the vnni512 gate and iq2xxs alone stays (1828-1924 + against the panel's 2456). zen4's grid residue at the engine shape stands: iq2s 0.85, iq3s 0.78, iq2xs + 0.86, iq3xxs 0.90 - the vnni512 panel form's compute. + The frame realignment (every generated kernel's entry gets an alloca [64 x i8] align 64, so spill slots + never straddle a cache line whatever RSP the caller arrives with): zen2 k6 heap phase 3344 / 3437 / 3737 / + 4389 (was 4.0-5.4 with two of three slow), engine phase 3423 / 3593 (reference 3683: 1.03-1.08x). The draw + narrowed, one outlier remains - the stack phase was part of it, not all of it. Round four, the row form with the COLUMN signs on zen4 (one thread, engine phase; reference in parentheses): iq2s 3541-3578 (3757: 1.05x), iq2xxs 2737-2771 (3922: 1.42x, from 0.89 with the table), iq3xxs 4234-4322 (5243: 1.22x, from 0.56 with the table; the panel 0.88), iq3s 6238-6309 (6105: 0.97), From 4a611cd7d479087c0031072996eab1fa0e9b47b7 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:46:53 -0700 Subject: [PATCH 062/123] grid decode: iq2s leaves the zen4 row form (3257 vs the panel's 2525 us at 16 lanes); the bench's team arm burns in before timing With iq2s second in the process its 16-lane row form read 3257 us against the panel's 2525 - the one-thread win (3600 vs 4918) does not survive the engine shape on zen4 - so the x86-vnni512 class keeps only iq2xxs on the row form. The slow slot that every A/B showed on the first format (iq2s first: 4887 with an 8.7 ms slot; iq2xxs first: 4063 with a 7.9 ms slot; the same format second: flat) is a first-format effect, not a kernel's: the team arm now runs six warmup dispatches per row before it times. TEST 90/90. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das | 6 +++++- modules/dasLLAMA/dasllama/dasllama_gemm_gen.das | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index a34873acaa..e0265e8ad3 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -251,6 +251,8 @@ struct RowStat { lane_hi : int64 // --team: the slowest } +let TEAM_BURN = 6 // warmup dispatches per row before the team arm times anything + //! the engine's decode dispatch, verbatim (dasllama_math's matmul shape): the splitter's chunk count over the //! rows, the team self-serving them; each lane adds its call time to laneP[slot] def team_gemv(fn : GemvFn; var yp : float?; kqp, ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d : int64; var laneP : int64?) { @@ -292,7 +294,9 @@ def time_rows_team(var stats : array; rounds : int; blk : block<(row : for (row in range(length(stats))) { stats[row].us |> reserve(rounds) stats[row].lane_lo = 999999999l - invoke(blk, row, laneP) // warmup + codegen, unmeasured + for (_w in range(TEAM_BURN)) { // codegen, then a burn: the first format's team rows read one slot at 8 ms without it + invoke(blk, row, laneP) + } } for (_r in range(rounds)) { for (row in range(length(stats))) { diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 5801176524..8c7e8c135c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1473,7 +1473,7 @@ def private grid_rows_path(te : TileEmit) : bool { if (knob == "1") return true if (knob == "0") return false let cls = tune_cpu_class() - if (cls == "x86-vnni512") return te.kq == 23 || te.kq == 25 // zen4: iq2s, iq2xxs (iq3xxs wins alone, loses at 16 SMT lanes) + if (cls == "x86-vnni512") return te.kq == 25 // zen4: iq2xxs (iq2s and iq3xxs win alone, lose at 16 lanes) if (cls == "x86-amx") return te.kq == 25 || te.kq == 34 // Granite Rapids: iq2xxs 1.47x, iq3xxs 1.36x at the engine shape return false } From 1e9a1a5b9b7e1e3f40371c4a6e7bd7cbc3983716 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 09:47:58 -0700 Subject: [PATCH 063/123] kernel parity plan: the burn-in verification and the final class gate --- plans/kernel_parity_pass.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 4c92619fcd..e80e8e2bde 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -643,6 +643,9 @@ stays open. never straddle a cache line whatever RSP the caller arrives with): zen2 k6 heap phase 3344 / 3437 / 3737 / 4389 (was 4.0-5.4 with two of three slow), engine phase 3423 / 3593 (reference 3683: 1.03-1.08x). The draw narrowed, one outlier remains - the stack phase was part of it, not all of it. + Verified after the burn-in (8693a5b47), zen4 16 lanes with iq2s FIRST: iq2s 2535 (the ladder's 2525; reference + 2159 -> 0.85), iq2xxs 1696 (row form; 2134 -> 1.26x), iq2xs 2309 (2000 -> 0.87). Gate final: x86-vnni512 -> + iq2xxs; x86-amx -> iq2xxs, iq3xxs; everything else the panel; the knob forces either. Round four, the row form with the COLUMN signs on zen4 (one thread, engine phase; reference in parentheses): iq2s 3541-3578 (3757: 1.05x), iq2xxs 2737-2771 (3922: 1.42x, from 0.89 with the table), iq3xxs 4234-4322 (5243: 1.22x, from 0.56 with the table; the panel 0.88), iq3s 6238-6309 (6105: 0.97), From 21bb51c49631bc4d9ad4d0b20ee9ef98453fee6e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:21:49 -0700 Subject: [PATCH 064/123] kernel parity plan: the next session's rulings - the gemv races the tile's top two, x86-amx stays split, re-mint every class --- plans/kernel_parity_pass.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index e80e8e2bde..f0ec729b68 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -471,10 +471,15 @@ the gemv inheriting the tile's crown (tune_companion). perf stat gave nothing (n ## 4. Work queue (one item at a time; a row is done at >= 1.0x on rig 2's shape AND the vehicle) -QUEUED (from the three-box rounds of 2026-09-01): the gemv's own crown. The tuner races the TILE and the gemv -follows (`tune_companion`); on zen4 and Intel the gemv prefers `vpdpbusd_width256` where the tile crowns -width 512 (iq2s 3-6%, k4 similar) - the mint should race the gemv seats at n=1 in the same walk and stamp -them separately. A tune-framework change (llvm_tune + the seat declarations), a mint per class after it. +NEXT SESSION (Boris's rulings, 2026-09-01 evening): the gemv's own crown. The tuner races the TILE and the gemv +follows (`tune_companion`); on zen4 and Intel the gemv prefers `vpdpbusd_width256` where the tile crowns width +512 (iq2s 3-6%, k4 similar), and on Intel the stamped k6 gemv runs 1.5x slower than the same seat raced. The mint +races the gemv at decode shape between the TILE'S TOP TWO seats only (not the whole grid - "i don't see the point +of racing anything but top 2") and stamps the gemv separately; more mint time is fine ("performance matters +more"); `x86-amx` stays a class of its own, split from `x86-vnni512` ("for sure"). Then re-mint every class on +the emitter it ships with: x86-avx2 on zen2, x86-vnni512 and x86-amx rented (aws_bootstrap.sh, 35 min each), +arm-i8mm on the M4, arm-neon on the M1. After that: the zen4 grid kernels (the vnni512 panel form's compute), then +Vulkan gap 1. The CPU PR follows the Metal PR's merge (the make_pr chain on this branch). CPU decode (gap 2) - the k-quant decode kernels were the tails the full ladder exposed (k3 0.80x, k6 0.90x, k5 0.91x): From a8ba4698bdb27d1c04a843a8e01198047cd7816b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:33:37 -0700 Subject: [PATCH 065/123] tune: the gemv companion gets its own seat - raced among the tile winner's same-layout top two, stamped from its own manifest entry The tile's crown was the gemv's by construction (tune_companion stamps the sibling with the main entry's perm), and on zen4 and Granite Rapids the gemv's fastest seat was the tile's runner-up. The plane layout the layout companion pins (mr) is shared, so a gemv can only take a seat with the tile winner's mr: the kq grids gain a width-256 row on the mr16 layout (dot=vpdpbusd, width=256, mr=16 - the same-layout alternative to the 512 crown), the harness picks the gemv among the tile winner's same-mr rows (the two best by tile time) by the streamed-decode time, only past the margin over the tile seat's own gemv and with the incumbent keeping a tie, and writes it under the companion's own name (q8_gemv_gen); the framework stamps a companion from its own entry when one names an eligible perm of the grid, else it follows the main as before. skills/tune.md carries the rule; LAWS.md the ruling. TEST 90 -> 103 rows ok (the new rows decline on zen2, no avx512vnni). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_math_gen.das | 14 ++++++ modules/dasLLAMA/harness/gen_tune_probe.das | 46 +++++++++++++++++-- modules/dasLLVM/daslib/llvm_tune.das | 31 ++++++++++++- skills/LAWS.md | 5 ++ skills/tune.md | 8 ++++ 5 files changed, 98 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 072d225b45..cdc72fdcd6 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -532,6 +532,7 @@ def q40q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "k4q8_gemv_gen", gen = "dasllama_gemm_gen::k4_gemv"), tune_companion(fn = "k4q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k4_tile", @@ -552,6 +553,7 @@ def k4q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "k5q8_gemv_gen", gen = "dasllama_gemm_gen::k5_gemv"), tune_companion(fn = "k5q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k5_tile", @@ -572,6 +574,7 @@ def k5q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "k6q8_gemv_gen", gen = "dasllama_gemm_gen::k6_gemv"), tune_companion(fn = "k6q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k6_tile", @@ -592,6 +595,7 @@ def k6q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "q40q8_gemv_gen", gen = "dasllama_gemm_gen::q40_gemv"), tune_companion(fn = "q40q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q40_tile", @@ -624,6 +628,7 @@ def iq4xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "iq4xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq4xs_gemv"), tune_companion(fn = "iq4xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq4xs_tile", @@ -656,6 +661,7 @@ def k3q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "k3q8_gemv_gen", gen = "dasllama_gemm_gen::k3_gemv"), tune_companion(fn = "k3q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k3_tile", @@ -688,6 +694,7 @@ def iq3sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "iq3sq8_gemv_gen", gen = "dasllama_gemm_gen::iq3s_gemv"), tune_companion(fn = "iq3sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3s_tile", @@ -763,6 +770,7 @@ def iq3xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "iq3xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq3xxs_gemv"), tune_companion(fn = "iq3xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3xxs_tile", @@ -826,6 +834,7 @@ def iq4nlq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "iq4nlq8_gemv_gen", gen = "dasllama_gemm_gen::iq4nl_gemv"), tune_companion(fn = "iq4nlq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq4nl_tile", @@ -895,6 +904,7 @@ def k2q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "k2q8_gemv_gen", gen = "dasllama_gemm_gen::k2_gemv"), tune_companion(fn = "k2q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k2_tile", @@ -967,6 +977,7 @@ def iq2sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "iq2sq8_gemv_gen", gen = "dasllama_gemm_gen::iq2s_gemv"), tune_companion(fn = "iq2sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2s_tile", @@ -1038,6 +1049,7 @@ def iq2xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "iq2xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xs_gemv"), tune_companion(fn = "iq2xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xs_tile", @@ -1111,6 +1123,7 @@ def iq2xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "iq2xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xxs_gemv"), tune_companion(fn = "iq2xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xxs_tile", @@ -1307,6 +1320,7 @@ def q51q8_gemv_gen(var yp : float?; qgp : uint8 const?; sgp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown tune_companion(fn = "q51q8_gemv_gen", gen = "dasllama_gemm_gen::q51_gemv"), tune_companion(fn = "q51q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q51_tile", diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 3eaa4ea0b1..86e8b30cff 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -446,6 +446,34 @@ def kq_ssb(fmt : int64) : int64 => dasllama_gemm_schema::kq_ssb(int(fmt)) // the family's sidecar entry = its [tune] function name (the K-quants by bit width - k3 included - q40/iq4xs by tag) def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt == 44l ? "iq4xsq8_tile_gen" : (fmt == 33l ? "iq3sq8_tile_gen" : (fmt == 34l ? "iq3xxsq8_tile_gen" : (fmt == 45l ? "iq4nlq8_tile_gen" : (fmt == 23l ? "iq2sq8_tile_gen" : (fmt == 24l ? "iq2xsq8_tile_gen" : (fmt == 25l ? "iq2xxsq8_tile_gen" : "k{fmt}q8_tile_gen"))))))) +//! the gemv companion's own manifest entry: the tile's name with the tier swapped +def kq_gemv_entry(fmt : int64) : string => replace(kq_tile_entry(fmt), "_tile_gen", "_gemv_gen") + +//! the gemv's seat: among the rows sharing the tile winner's layout (mr), the two best by tile time, the one +//! with the best streamed-decode time takes the gemv - only by the margin over the tile winner's own gemv, +//! and the incumbent keeps a tie; "" when the tile winner keeps it (no separate entry, the companion follows) +def private kq_gemv_seat(fmt : int64; winner : int; names : array; vmr : array; tile, gemv : array) : string { + var cand <- [for (i in range(length(names))); i; where vmr[i] == vmr[winner] && tile[i] < 1.0e29lf] + cand |> sort() $(a, b) => tile[a] < tile[b] + if (length(cand) > 2) { + cand |> resize(2) + } + var pick = winner + for (c in cand) { + if (gemv[c] < gemv[pick]) { + pick = c + } + } + let m = gen_margin() + if (pick != winner && gemv[winner] <= gemv[pick] * m) { + tune_detail("{kq_gemv_entry(fmt)}: {names[pick]} does not beat the tile seat {names[winner]} on the gemv by x{m:0.3f} - the gemv follows the tile\n") + pick = winner + } + pick = gen_sticky(kq_gemv_entry(fmt), names, gemv, pick) + delete cand + return pick == winner ? "" : names[pick] +} + // synthetic disk superblocks, varied per (row, superblock) so no two blocks repeat: any byte // is a legal quant, and the small d/dmin keep the f16 (s, o) pairs finite and row sums tame def pack_kq_scale_header(var blkb : array; base : int) { @@ -1253,7 +1281,7 @@ def kq_test_family(fmt : int64; kfxs : array) : bool { // and the gemv (streamed + hot decode shapes), print the table, and return the tile-best // suffix ("" = a gate failed). The kq gemv is nrsplit-independent — same-mr rows share the // plane and the gemv shape — so the tile bench decides the family entry. -def kq_tune_family(fmt : int64; var rows_out : table) : string { // nolint:STYLE038 — one family race: shared repacked buffers couple gate, bench and pick +def kq_tune_family(fmt : int64; var rows_out : table; var gemv_out : string&) : string { // nolint:STYLE037,STYLE038 — one family race: shared repacked buffers couple gate, bench and the two picks // batch shape: fat STREAMING ffn (weights > the L2 budget, multi-token-block walk) — the shape // class that carries ~90% of prefill time. The old 2048x512x64 kv-projection probe was L2-hot // and crowned hot-shape winners that lose double-digit % at model scale (mr4 vs mr8, M1). @@ -1379,6 +1407,10 @@ def kq_tune_family(fmt : int64; var rows_out : table) : stri } var tnames <- [for (t in tvs); t._0] winner = gen_sticky(kq_tile_entry(fmt), tnames, best, winner) + gemv_out = kq_gemv_seat(fmt, winner, tnames, vmr, best, gbest) + if (!empty(gemv_out)) { + rows_out["gemv_seat"] = JV((seat = gemv_out, tile_seat = tvs[winner]._0, gemv_us = gbest[winner])) + } delete tnames let wname = tvs[winner]._0 delete keep @@ -2088,8 +2120,9 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - // the kq families tune separately — one manifest entry per format, tile-best wins (the - // gemv is nrsplit-independent, so same-mr rows share its plane and kernel shape). No + // the kq families tune separately — the tile-best row is the format's entry, and the gemv gets its + // own entry when a same-layout row beats the tile seat on the streamed decode shape (the layout + // companion pins the plane mr, so only same-mr rows can differ; the two best by tile time race). No // child-process confirm gate here: the kq stamp only moves the kq planes' interleave, // and the e2e exposure (a tile-optimal mr shaving decode kernel rate) is bounded by the // decode path staying DRAM-bound — validated end-to-end when the entries first landed. @@ -2099,13 +2132,18 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family let entry = kq_tile_entry(fmt) tune_progress_kernel_begin(entry, ROUNDS) var krows : table - let w = kq_tune_family(fmt, krows) + var gw = "" + let w = kq_tune_family(fmt, krows, gw) tune_detail("TUNE_GEN_TIME {entry} family {get_time_usec(kq_t0) / 1000} ms\n") tune_progress_kernel_end(entry, w, empty(w) ? "rejected" : "beats") // gate failure (the broken perm is named above): queue the always-correct reference // body so the scope still COMPLETES — a missing key re-tunes every start pending[entry] = empty(w) ? "reference" : w print("{entry} winner: {empty(w) ? "reference (gate-failure fallback)" : w}\n") + if (!empty(gw)) { // the gemv's own seat (same layout as the tile's); absent = the companion follows the tile + pending[kq_gemv_entry(fmt)] = gw + print("{kq_gemv_entry(fmt)} winner: {gw} (the tile seat {w} keeps the tile)\n") + } g_rows[entry] = JV(krows) } if (!noise_gate("mid2")) { diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index 1818680066..d55a51c8a9 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -1653,6 +1653,18 @@ class private TuneAnnotation : AstFunctionAnnotation { suffix = "" } } + var csuffix : array // a companion's OWN entry (the harness's gemv seat) when it names an eligible perm of this grid; else it follows the main + csuffix |> resize(length(comps)) + if (manifestFound) { + for (ci, c in count(), comps) { + let own = manifest?["{c.fnname}"] ?? "" + for (p in perms) { + if (p.suffix == own && perm_requires_ok(p)) { + csuffix[ci] = own + } + } + } + } let fromManifest = !empty(suffix) var ok = true if (empty(suffix)) { @@ -1685,8 +1697,22 @@ class private TuneAnnotation : AstFunctionAnnotation { for (p in perms) { if (p.suffix == suffix) { stamp_llvm_code(func, genKey, p, fromManifest ? mpath : "") - for (c in comps) { - stamp_llvm_code(c.fn, c.genkey, p, fromManifest ? mpath : "") + for (ci, c in count(), comps) { + var own = false + if (!empty(csuffix[ci]) && csuffix[ci] != suffix) { + for (q in perms) { + if (q.suffix == csuffix[ci]) { + stamp_llvm_code(c.fn, c.genkey, q, mpath) + own = true + } + } + if (own && tune_verbosity() == "verbose") { + print("llvm_tune: {c.fnname} <- {csuffix[ci]} (its own entry, {mpath})\n") + } + } + if (!own) { + stamp_llvm_code(c.fn, c.genkey, p, fromManifest ? mpath : "") + } } if (tune_verbosity() == "verbose") { print("llvm_tune: {fname} <- {suffix} ({fromManifest ? mpath : "fallback"})\n") @@ -1709,6 +1735,7 @@ class private TuneAnnotation : AstFunctionAnnotation { } } } + delete csuffix delete perms delete comps return ok diff --git a/skills/LAWS.md b/skills/LAWS.md index 2d5713a478..4e75e466ff 100644 --- a/skills/LAWS.md +++ b/skills/LAWS.md @@ -23,3 +23,8 @@ then there is a race. if nothing available - no race." Profiles are keyed by CPU it), and separate CPU from GPU: "do u want to separate CPU and GPU tunes?" - resolved as kernels-only profiles, runtime knobs never travel. He will remove the dasllama.io tune service if the new-silicon race drops to a couple of minutes. +## 2026-09-01 - skills/tune.md, the `[tune_companion]` section + +The gemv companion gets its own manifest entry, raced among the tile winner's same-layout rows, top two by tile time +only: "i don't see the point of racing anything but top 2"; "im fine with more mints. performance matters more"; +"we split x64-amx from x86-vnni512 for sure". diff --git a/skills/tune.md b/skills/tune.md index 80aa9375a9..1ca46eef6a 100644 --- a/skills/tune.md +++ b/skills/tune.md @@ -89,6 +89,14 @@ two-function stamp, so a kernel and (say) its repack-layout query can never desync, JIT-time declines included. The sibling is a plain function declared earlier in the same module, with an explicit return type. +A companion may carry its **own** manifest entry, keyed by its function name: +when that entry names a permutation of the same grid that this box can run, +the companion stamps it instead of the main entry's. The harness writes one +only where it measured a different winner for the sibling - the dasLLAMA tuner +gives the kq gemv its own seat among the rows sharing the tile winner's plane +layout - and never for a companion whose choice must follow the main (the +layout query). No entry = the companion follows, as before. + ## The mode contract The `DAS_TUNE_MODE` environment variable selects the compile-time behavior: From 11eba92289b276a69a468d7654774dbddf73dfc1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:36:23 -0700 Subject: [PATCH 066/123] kernel parity plan + sidecar how-to: the gemv's own seat landed, the re-mint round --- modules/dasLLAMA/HOW_TO_GET_SIDECAR.md | 3 ++- plans/kernel_parity_pass.md | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md index 5f12422391..7bf9b76178 100644 --- a/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md +++ b/modules/dasLLAMA/HOW_TO_GET_SIDECAR.md @@ -161,7 +161,8 @@ Read the sidecar's provenance before anything else: python3 -c 'import json;d=json.load(open("modules/dasLLAMA/benchmarks/lcpp_bench.tune.json"));p=d["provenance"];print(p["noise"],p["validation"],p["features"],len(d["kernels"]))' ``` -`ok ok 49` is the pass: `noise` is the tuner's own drift verdict (a busy or thermally +`ok ok 49` is the pass (a few more when the harness gave a kq gemv its own seat - those entries are +named `q8_gemv_gen` and ride into the profile like any other): `noise` is the tuner's own drift verdict (a busy or thermally unstable box says otherwise - re-mint, never edit), `validation` is every winner checked against its fallback, `features` is the box's fingerprint of `TUNE_KNOWN_FEATURES` (zen4: `avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni`), 49 the kernel count the scope demands. diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index f0ec729b68..a85ba7fb24 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -480,6 +480,15 @@ more"); `x86-amx` stays a class of its own, split from `x86-vnni512` ("for sure" the emitter it ships with: x86-avx2 on zen2, x86-vnni512 and x86-amx rented (aws_bootstrap.sh, 35 min each), arm-i8mm on the M4, arm-neon on the M1. After that: the zen4 grid kernels (the vnni512 panel form's compute), then Vulkan gap 1. The CPU PR follows the Metal PR's merge (the make_pr chain on this branch). +LANDED (0251e8b2d): the kq grids gain `dot=vpdpbusd width=256 mr=16` (the same-layout alternative to the 512 tile +crown - the layout companion pins the plane mr, so only same-mr rows can differ); the harness picks the gemv among the +tile winner's same-mr rows, the two best by tile time, by the streamed-decode time past the margin over the tile +seat's own gemv (incumbent keeps a tie) and writes it under `q8_gemv_gen`; llvm_tune stamps a companion from +its own entry when it names an eligible perm of the grid. TEST 90 -> 103 rows on zen2 (the new rows decline there). +The arm classes and x86-avx2 have no same-mr alternative (their grids collapse to one eligible row), so only +x86-vnni512 and x86-amx need the re-mint: two rentals with `aws_bootstrap_mint.sh` (build, TEST, the 1B vehicle's +--tune, export, both ladders) - running. The export copies every kernels entry, so the gemv seats ride into the +class profiles; the profile adoption merges them as ordinary entries. CPU decode (gap 2) - the k-quant decode kernels were the tails the full ladder exposed (k3 0.80x, k6 0.90x, k5 0.91x): From 7eb39c02484d81470dfb3b176102f356ccb1b7c2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:42:55 -0700 Subject: [PATCH 067/123] research memo: llama.cpp's grid kernels on zen4 - no AVX-512 path, the panel round trip is our stall, the VBMI symbol lattice is the lever --- plans/kernel_parity_research_zen4.md | 289 +++++++++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 plans/kernel_parity_research_zen4.md diff --git a/plans/kernel_parity_research_zen4.md b/plans/kernel_parity_research_zen4.md new file mode 100644 index 0000000000..1bbd1e07b5 --- /dev/null +++ b/plans/kernel_parity_research_zen4.md @@ -0,0 +1,289 @@ +# zen4 grid decode: why llama.cpp wins there and loses on zen2 + +Read-only study of the five codebook formats (IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S; our +kq ids 25/24/23/34/33) at the n=1 decode shape, on AMD EPYC 9R14 (zen4, AVX-512 + VNNI, +our class `x86-vnni512`) against zen2 (3990X, AVX2, class `x86-avx2`). + +Trees read: `D:\Work\llama.cpp` at 6c84c7d5d (`ggml/src/ggml-cpu/arch/x86/quants.c`, +`arch/x86/repack.cpp`, `repack.cpp`, `ggml-cpu.c`, `ggml-common.h`, `ggml-cpu/CMakeLists.txt`, +and the MSVC object `build-clean-cpu/ggml/src/ggml-cpu.dir/Release/ggml-cpu/arch/x86/quants.c.obj`); +`D:\Work\daScript-iquant` (`dasllama_gemm_gen.das`, `dasllama_repack.das`, `llvm_tune.das`, +`llvm_jit_common.das`). Nothing modified in either tree. `x86:N` = `arch/x86/quants.c` line N. + +--- + +## 0. The answer in four lines + +llama.cpp's five grid kernels have **no AVX-512 branch at all** - a znver4 build compiles the +same `#if defined(__AVX2__)` body it compiles on zen2 (x86:2673, 2790, 3088, 3273, 3397). Their +1.42-1.90x zen2 -> zen4 gain is pure microarchitecture on unchanged code: their kernel is a wall +of independent 64-bit table loads, and zen3/zen4 went from 2 loads/cycle to 3 and grew the OOO +window. Their IPC rises from 2.2-3.9 to 3.8-5.9. Ours gains only 1.14-1.26x because the panel +form is **stall-bound, not instruction-bound** - we already execute fewer instructions per weight +than they do and still take longer. + +--- + +## 1. What a zen4 build actually runs + +- Flags: `GGML_NATIVE=ON` -> `-march=native` on gcc/clang (`ggml-cpu/CMakeLists.txt:305-306`); + the local zen2 reference is MSVC (`build-clean-cpu/CMakeCache.txt`: Visual Studio generator, + `/O2`, `GGML_AVX512:BOOL=OFF`) and goes through the FindSIMD probe at CMakeLists.txt:247-296. +- **All five vec_dots are `#if defined(__AVX2__)` / `#elif defined(__AVX__)` / generic.** There + is no `__AVX512F__` and no `__AVX512VNNI__` arm anywhere in them: iq2_xxs x86:2673, iq2_xs + x86:2790, iq2_s x86:3088, iq3_xxs x86:3273, iq3_s x86:3397. On znver4 `__AVX2__` is defined, + so the AVX2 body wins and every vector is 256 bits. +- The one VNNI-aware helper in the file, `mul_sum_us8_pairs_float` (x86:105-119, `_mm256_dpbusd_epi32` + under `__AVX512VNNI__ && __AVX512VL__`), is **not called by any of the five** - they all call + `_mm256_maddubs_epi16` directly (x86:2699, 2887, 3145, 3303, 3464) because the per-block scale + is folded into the following `_mm256_madd_epi16`, which VNNI cannot do. +- **No repack GEMV for any of the five.** `ggml_repack_get_optimal_repack_type` + (`repack.cpp:4528-4722`) covers Q4_0, Q4_K, Q2_K (AVX-512 only, :4627-4632), Q5_K, Q6_K, + IQ4_NL, MXFP4, Q8_0 and nothing else; `grep IQ2\|IQ3` over `repack.cpp` and + `arch/x86/repack.cpp` is empty. `llamafile_sgemm` covers only IQ4_NL among the iq types + (`sgemm.cpp:4131`). Type traits are `nrows = 1`, `vec_dot_type = Q8_K`, `from_float = NULL` + (`ggml-cpu.c:336-366`), so real serving takes the per-row `vec_dot` above. **The grid rows of + the ladder are the fair ones** - unlike k4/k5/k6/q4_0/q8_0/iq4_nl/q2_K. + +--- + +## 2. Instruction shape per 32 weights of ONE row + +Counted from the MSVC `/arch:AVX2` object's inner loops (the zen2 reference binary; the zen4 +build compiles the same source with gcc, so counts shift but the shape does not). + +| fmt | loop | weights/iter | instr | per 32w | mem ops | vec instr | scalar instr | +|---|---|---|---|---|---|---|---| +| iq2_xxs | 0xb1..0x26f | 64 | 85 | 42.5 | 33 | 34 | 51 | +| iq2_xs | 0x120..0x3cb | 128 | 110 | **27.5** | 34 | 100 | 10 | +| iq2_s | 0x150..0x2f2 | 64 | 90 | 45.0 | 26 | 34 | 56 | +| iq3_xxs | 0xc1..0x2e2 | 64 | 96 | 48.0 | 43 | 42 | 54 | +| iq3_s | 0x110..0x39f | 64 | 119 | **59.5** | 31 | 84 | 35 | + +Per 32 weights, by stage: + +- **IQ2_XXS** (x86:2687-2706): 4 `movzbl` index bytes; 4 x u64 `iq2xxs_grid[]` and 4 x u64 + `keven_signs_q2xs[]` reads, all folded into `vmovq`/`vpinsrq` pairs plus 2 `vinsertf128` + (8 `vmovq` + 8 `vpinsrq` + 4 `vinsertf128` per 64 weights); 4 `shr` + 4 `and` for the 7-bit + sign codes; 1 `vpsignb` on the ACTIVATION; 1 `vpmaddubsw`; scale `2*ls+1` as `vmovd` + + `vpbroadcastw` + 1 `vpmaddwd`; 1 `vpaddd`. 16.5 memory ops, of which 8 are 64-bit table reads. +- **IQ2_XS** (x86:2841-2900) is the outlier and the fastest: the index extraction is vectorized + (`_mm256_and_si256(q2_data, m511)`, x86:2842) and the 8th sign bit is a **4-bit parity by + `vpshufb`** (x86:2844-2849, the `k_bit_helper` table) instead of a `ksigns` read. MSVC pulls the + 16 indices back out with 14 `vpextrw` rather than the source's stack union (x86:2823-2824). + Only 8.5 memory ops per 32 weights, 4 of them u64 grid reads. Sign expansion is + `vpshufb`+`vpand`+`vpcmpeqb`+`vpor`+`vpsignb` (x86:2871-2885), 5 vector ops per 32 bytes. +- **IQ2_S** (x86:3120-3151): index arithmetic in the ADDRESS expression (`qs[3] | ((qh[ib32]<<2) + & 0x300)`, x86:3123-3130) - 10 `movzbl`, 11 `shl`, 10 `or`, 8 `and` per 64 weights; 4 u64 grid + reads per 32 weights; signs from an explicit plane byte via `vpbroadcastd`+`vpshufb`+`vpand`+ + `vpcmpeqb` and the cheap **`sub(xor(mask,q8), mask)`** conditional negate (x86:3136) - 2 ops, + not `or(mask,1)`+`vpsignb`. +- **IQ3_XXS** (x86:3287-3310): the most memory ops of the five (24.5 per 32 weights) because its + grid entry is only 4 weights: **8 dword** `iq3xxs_grid[]` reads per 32 weights (16 `movzbl` + + 12 `vpinsrd` + `vmovd` per 64 weights) plus 4 u64 sign reads. +- **IQ3_S** (x86:3429-3471): the most instructions. The 9-bit index is built in a vector + (`_mm256_sllv_epi32` + `_mm256_or_si256`, x86:3433-3438) and then taken back out - MSVC uses 12 + `vpextrd` where the source writes a `__m256i`/`uint32_t[16]` union (x86:3413-3418). 8 dword grid + reads + 12 `vpinsrd` per 64 weights, then the same `sub(xor)` sign negate. + +Everything is 256-bit. Nothing is 512-bit. Nothing uses `vpdpbusd`. + +--- + +## 3. Why zen4 favors that shape, and where our panel loses + +Machine facts (Zen 2 -> Zen 4): macro-op **dispatch stayed 6/cycle**; **loads went 2/cycle -> +3/cycle** and stores 1 -> 2; the ROB grew 224 -> 320; AVX-512 ops are executed as two 256-bit +passes over the same four FP pipes, so a 512-bit instruction has the same bytes/cycle as two +256-bit ones and only saves front-end slots. + +Derived cycles per 32 weights and IPC (clock assumed 4.2 GHz zen2 / 3.7 GHz zen4 - **estimates**; +m=4096 k=14336 = 1.835e6 groups of 32 weights): + +| fmt | ref zen2 us / c32 / IPC | ref zen4 us / c32 / IPC | ours zen2 c32 | ours zen4 c32 | ratio gain ref / ours | +|---|---|---|---|---|---| +| iq2s | 5022 / 11.5 / 3.9 | 3757 / 7.6 / **5.9** | 11.6 | 9.9 | 1.51 / 1.17 | +| iq2xs | 5354 / 12.3 / 2.2 | 3624 / 7.3 / 3.8 | 10.6 | 9.2 | 1.68 / 1.16 | +| iq2xxs | 5103 / 11.7 / 3.6 | 3922 / 7.9 / 5.4 | 11.9 | 9.8 | 1.48 / 1.21 | +| iq3s | 10242 / 23.4 / 2.5 | 6105 / 12.3 / 4.8 | 17.4 | 15.2 | 1.90 / 1.14 | +| iq3xxs | 6573 / 15.0 / 3.2 | 5243 / 10.6 / 4.5 | 15.2 | 12.1 | 1.42 / 1.26 | + +Readings: + +1. **Their kernel was load-issue and window limited on zen2 and is near the dispatch ceiling on + zen4.** IQ2_XXS issues 33 memory ops per 64 weights; at 2 loads/cycle that is a 16.5-cycle + floor against 85 instructions / 6 = 14.2 cycles of dispatch, and it measured 23.4 - short of + both, i.e. window-limited. On zen4 the load floor drops to 11 cycles and it measures 15.8 + against the same 14.2 dispatch bound: **90% of the machine's issue width.** They cannot get + much more from instruction selection on zen4; only from fewer instructions. +2. **We are not instruction-bound - we are stall-bound.** Our iq2xxs panel emits, per row per + superblock, 32 index byte loads + 32 shl + 32 u64 grid loads + 32 lshr + 64 stores (about 24 + instructions per 32 weights, `emit_kq_gather` gemm_gen.das:1585-1616, `iq2_idx2` :1421-1435, + `grid_pair_half` :1381-1385), plus about 7 for the dot and 2 for the sign column: ~33 per 32 + weights against their 42.5. We run **fewer** instructions in **more** cycles - IPC ~3.4 + against their 5.4 (estimate). Nothing zen4 widened is our constraint. +3. **The panel is the only structure we have that they do not.** At mr=16 it is 4096 bytes; the + gather writes 1024 scattered 4-byte stores per superblock-group, one per row into each of 64 + distinct cache lines, visiting the same 64 lines 16 times, and the dot then reads them back as + 64-byte vectors each spanning 16 of those stores (`panel_group_off`, gemm_gen.das:1577-1580; + the reload in `emit_block_iq4xs`). Store issue alone is 512 of the measured ~1253 cycles per + superblock-group. Zen4 doubled store issue (1 -> 2/cycle) and we still gained only 1.21x, so + the cost is the round trip, not the store throughput - a wide load over many narrow stores is + the one direction x86 does not forward. Their own memory round trips (x86:2823 `aux_gindex`, + x86:3413 `index_t`) are wide-store -> narrow-load and carry indices, never weights. +4. **Why the row form (`grid_rows_path`, gemm_gen.das:1469-1479) wins on some formats only.** It + replaces the panel with one u64 grid load per row per 8 weights and a `vmovq`/`vpinsrq`/ + `vinserti128`/`vinserti64x4` tree (`row_weights8` :1749-1759, `concat_rows` :1713-1719). The + whole 8-row group's vector cannot issue until the slowest row's index chain finishes, so the + form trades the panel's stall tolerance for a shorter but **exposed** dependency chain. It wins + exactly where the per-row index is one byte with no arithmetic: + - iq2xxs (1 index byte, `wg_byte` at column 2*blk): 4866 -> 2737 us = 5.5 c32, **ahead of + their 7.9** - already seated for `x86-vnni512` (gemm_gen.das:1476). + - iq3xxs (1 index byte per 4 weights, column signs): 5973 -> 4234 = 1.41x. Seated on + `x86-amx` only (:1477), **not on vnni512**. + - iq2s / iq3s (qs byte + qh bits: `shl`+`and`+`or` on the chain): 4918 -> 3559-3656 and + 7560 -> 6267 - win alone, lose or go mixed at 16 lanes. + - iq2xs (u16 assembled from two byte loads, `iq2xs_word16` :1411-1417, and the sign code in + the same word): 4558 -> 5381 - loses. +5. **SMT flips the verdict.** Two lanes per core share the four FP pipes and one 32 KB L1D. A + stall-bound kernel gets its bubbles filled by the sibling, an instruction-bound one does not, + so at 16 lanes the panel catches up and the row form's extra instructions start to cost: our + one-thread ratios 0.76-0.88 become 0.84-0.90 at the engine shape for four of five formats, and + iq2s's row form "wins alone, loses at 16 lanes" is exactly this. Corollary for every item + below: **instruction cuts pay less at 16 lanes than at one thread, and plane-size increases + cost more** (iq3s already pulls 49 GB/s of the box's 87 at 16 lanes). +6. **The zen2 -> zen4 comparison of THEIR numbers mixes compilers** (MSVC `/arch:AVX2` vs gcc + `-march=znver4`). Part of their 1.42-1.90x is codegen, not silicon. It does not change our + conclusions - ours is measured on our own emitter on both boxes - but do not quote their gain + as a microarchitecture number without rebuilding one side. + +--- + +## 4. Ranked spellings for the vnni512 lattice + +Every item is class-gated, so zen2 keeps what it has unless stated. + +**1. Seat the row form for iq3xxs on `x86-vnni512`.** One line: extend gemm_gen.das:1476 from +`te.kq == 25` to `te.kq == 25 || te.kq == 34`, matching what x86-amx already does (:1477). +Measured on the box: 5973 -> 4234 us, i.e. 0.88x -> **1.24x** of the reference. zen2 untouched +(the class gate). 16 lanes: x86-amx recorded 1.36x at the engine shape for the same change, so +the SMT risk is low. Do this first. + +**2. `gather="qpanel"` - store the iq2 grid entry as ONE qword.** The three iq2 formats read a +u64 grid entry covering 8 weights and then split it into two dwords (`grid_pair_half` +:1381-1385) written to two panel groups 4*mr bytes apart. Re-lay the panel so a row's 8 +consecutive weights are contiguous, store the u64 unsplit, and let the dot read 8-rows-x-8-weights +vectors - the shape `grid_rows_dot` + `pairwise_add_i32` already consume (:1723-1745, :1801). +Per 8 weights the gather goes 1 idx load + 1 shl + 1 grid load + 1 lshr + 2 stores = 6 down to 4, +and **panel stores halve** (64 -> 32 per row per superblock), which is the resource item 3 named. +Targets iq2s, iq2xs, iq2xxs. Estimate 1.2-1.4x on those three at one thread, less at 16 lanes. +No new ISA, no plane change, helps zen2 identically. iq3s/iq3xxs get nothing (their entry is 4 +weights) and stay on item 1. + +**3. Bake the parity-completed sign byte at repack.** `kq_sign_bytes` (:1514-1548) still computes +`ksigns[i] == i | parity(i)<<7` at runtime with a 3-deep XOR/shift chain per `l` per block: +about 40 vector instructions per (blk, qd), 320 per superblock per row group. For **iq2xxs and +iq3xxs it is free** - the aux dword is 4 x 7 sign bits + a 4-bit scale, and the scale already +moved to the scale row, so four full sign bytes fit the same 4 bytes; for **iq2xs** it costs +1 +bit per 8 weights (9 index + 8 sign > 16), so race it. iq2s/iq3s already carry explicit sign +bytes (:1519-1521). Estimate 5-8% on the panel form and more on the row form, on every class and +both shapes; zen2 gains too. + +**4. Mask-register signs at width 512.** `apply_sign_col` (:1552-1564) and `sign_col_pm1` +(:1455-1463) build a byte mask with shuffle + `and` + `icmp` + `sext`, then `xor` + `sub` - six +vector ops per weight vector, each two uops at 512 bits. AVX-512 wants `vptestmb` into a +k-register plus a merge-masked `vpsubb`: two instructions. Emit the compare so LLVM keeps the +mask in `k` (do not `sext` it to `<64 x i8>`). Estimate 3-4 vector instructions saved per weight +vector, ~25% of the dot side. Width-256 path (zen2) unchanged - it keeps `llvm.x86.avx2.psign.b` +(:1730-1732), which is already one instruction. + +**5. Give the grid gemv a 256-bit seat on vnni512.** Measured in the pass's own races: iq2s at 16 +lanes 2528 (maddubs256) / **2383 (vpdpbusd256)** / 2530 (vpdpbusd512) - the crowned 512 seat +loses 6%. Mechanism, not noise: a 512-bit op on zen4 is two 256-bit passes over the same pipes +(no throughput gain), mr=16 doubles the panel to 4 KB per lane (8 KB per core under SMT, a +quarter of L1D), and width 512 forces the `icmp`/`select` sign path instead of the single +`vpsignb`. Blocked on the already-queued "gemv's own crown" item (the gemv currently inherits the +tile's `tune_companion`). Estimate 3-6%, larger at 16 lanes. + +**6. The VBMI symbol lattice - the big one, zen4/Granite/Ice Lake only.** Verified over +`ggml-common.h`: every grid byte comes from a tiny alphabet - `iq2xxs_grid` (:560), +`iq2xs_grid` (:627) and `iq2s_grid` (:758) use exactly **3** distinct bytes {0x08, 0x19, 0x2b}, +and `iq3xxs_grid` (:1017) and `iq3s_grid` (:1052) exactly **8** ({0x04,0x0c,0x14,0x1c,0x24,0x2c, +0x34,0x3e} and the odd 0x01..0x0f). So a grid entry is **16 bits** (8 weights x 2) for the iq2 +family and **12 bits** (4 weights x 3) for the iq3 family, and the entire decode can run in +vectors: +- The grp repack puts a 4-byte column's mr rows contiguous (`repack_iq2xxs_grp`, + dasllama_repack.das:663-700: `kq[dq + (c*mr + r)*4 + t]`), so at mr=16 **one 64-byte load is 64 + grid index bytes** (16 rows x 4 groups). +- For the **256-entry** grids (iq2xxs, iq3xxs) the compact code is two 256-byte byte-planes, and + a 256-byte byte-table lookup is 2 x `vpermi2b` + one blend on index bit 7: ~3 instructions per + 64 indices per plane, ~6-8 for both. +- Expand the 2-bit / 3-bit fields with `vpmovzxbd` + `vpmultishiftqb` (per-byte bit-offset + extraction) + `vpandq` + `vpshufb` against the 3- or 8-entry alphabet: **4 instructions per + 64-byte weight vector**, i.e. 16 rows x 4 weights. +- `ksigns_iq2xs` is `uint8_t[128]` (ggml-common.h:513) - exactly two zmm, so the sign-byte lookup + is ONE `vpermi2b` if item 3 is not taken. +Budget: ~5-7 vector instructions per 64-weight vector against today's ~24 scalar instructions per +32 weights per row. Estimate 2-3x on the five formats at one thread; at 16 lanes the win is +smaller (item 5 of sec.3) but positive, because it removes stores rather than adding traffic. +Prerequisites: add `avx512vbmi` to `TUNE_KNOWN_FEATURES` (llvm_tune.das:1235) and a gate beside +`g_target_x64_avx512vnni` (llvm_jit_common.das:626-628). zen2 declines the seat. +For the **512/1024-entry** grids (iq2xs, iq2s, iq3s) the compact table does not fit `vpermi2b` +(8-16 zmm of table); either carry the 12/16-bit code in the plane instead of the index (+1.33-1.5x +plane bytes, which at 16 lanes trades compute for bandwidth - measure before committing) or leave +those three on items 1-4. + +**7. Do not seat a hardware gather.** llama.cpp measured `_mm256_i32gather_epi32` losing to +`_mm256_set_epi32` **on a Ryzen 7950X, which is zen4**, and left the code commented out with the +note at x86:3440-3442. Zen4 `vpgatherdd` is still microcoded (order 10+ cycles per instruction - +estimate). The column read plus `vpermi2b` (item 6) is the vector path that works here. + +**8. Do not seat `vpdpbssd` or `bias=128` for this class.** Zen 4 has no AVX-VNNI-INT8; those +seats are Zen 5 / Sierra Forest coverage, not a zen4 fix. + +--- + +## 5. Traps + +1. **There is no AVX-512 reference kernel to copy.** All five are `#if defined(__AVX2__)` + (x86:2673, 2790, 3088, 3273, 3397); a znver4 build runs 256-bit `vpmaddubsw`, never + `vpdpbusd`. Anything you find that looks like a 512-bit grid kernel in ggml is for another type. +2. **`_mm256_set_epi64x(grid[a],grid[b],grid[c],grid[d])` is not a vector gather.** It is four + scalar table loads folded into `vmovq`/`vpinsrq` plus a `vinsertf128` - the disassembly shows + exactly that. Transposing it into our layout buys the store, not the loads; the win only + arrives when scalar table reads per weight go DOWN. +3. **Signs-on-the-activation does not port** (already ruled, research_cpu.md sec.8): our lanes are + different rows with different sign patterns while the activation is splatted. llama.cpp's own + arm64 arms put signs on the weights for the same reason. +4. **Their memory round trips run the safe way.** `aux_gindex` (x86:2823-2824, commented "somewhat + hacky, but gives a significant boost") and `index_t` (x86:3413-3418) are ONE wide store feeding + narrow fully-contained loads, and they carry indices. Our panel is the opposite direction and + carries weights. Do not "unify" them. +5. **The iq2_xs `vpshufb` parity trick (x86:2844-2849) is a workaround, not a win for us.** It + exists because x86 cannot vectorize a 128-entry x 8-byte table read; we can bake the + parity-completed byte at repack (item 3) and skip both the table and the parity. +6. **Their scale fold into `vpmaddwd` (x86:2703, 3307, 3468) does not port.** Their scale is one + scalar broadcast over a whole row; ours differs per lane and already rides the repacked scale + row with the global factor (0.125f/0.25f) pre-baked. Porting it is a regression. +7. **Do not reintroduce a horizontal reduction.** They pay `hsum_float_8` per row (x86:2713, 2907, + 3158, 3317, 3478); our accumulator lanes are the output rows. A decode that gets cheaper at the + price of a per-row reduction is a net loss. +8. **`nrc == 1` is asserted in all five** (e.g. x86:2662). There is no multi-row reference grid + kernel - our tile's 6.5-11.9x is against a kernel nobody wrote for that shape, and it is not + evidence about the decode. +9. **The dword column read is format- and form-specific.** `wg_col_byte` (gemm_gen.das:1364-1371) + deliberately falls back to a plain byte load under `te.gridRows` (:1365, "the row form measured + 1.8x slower on the dword read"), while the same read is a win in the panel form for iq3xxs + (:1404, 7862 -> 6602) and iq2xs (:1410, 6501 -> 4702). Do not unify them; race per (format, form). +10. **Count tables per core, not per lane.** Under SMT two lanes share one 32 KB L1D. The iq2s grid + is 8 KB, iq2xs 4 KB, `keven_signs_q2xs` 1 KB, and two mr=16 panels are another 8 KB. Any new + table (item 6 adds 512 B - 1 KB of compact code plus register-resident `vpermi2b` halves) has + to be budgeted against that, and a plane-size increase is charged at the DRAM ceiling + (87 GB/s on the c7a; iq3s already draws 49). +11. **The reference's zen2 and zen4 numbers come from different compilers** (MSVC `/O2 /arch:AVX2` + in `build-clean-cpu`, gcc `-march=native` on the AWS box). Before attributing any part of their + 1.42-1.90x to zen4, rebuild one side with the other toolchain. +12. **iq2_xs is their best format, not their worst** (27.5 instructions per 32 weights against + 42-60 for the rest) because it is the only one whose index extraction and parity are + vectorized. Expect it to be the hardest of the five to pass, and do not read its 0.80x as the + same problem as iq3s's 0.81x. From 9a9aa559efcd53bb5323c1fbee6f28f5e15ad34e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:53:24 -0700 Subject: [PATCH 068/123] kernel parity plan: the zen2 mint smoke, the zen4 research memo in the memo list --- plans/kernel_parity_pass.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index a85ba7fb24..9d32becb09 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -454,6 +454,18 @@ the gemv inheriting the tile's crown (tune_companion). perf stat gave nothing (n (3) a `vpdpbssd` seat (AVX-VNNI-INT8 hosts, no generator change); (4) repack-baked parity sign byte (free for IQ2_XXS/IQ3_XXS). Incidental waste: IQ2_XXS reads its aux word as four i8 loads + shifts where IQ3_XXS uses one i32 load; IQ2_XS builds a u16 from two i8 loads. +- `kernel_parity_research_zen4.md` - why llama.cpp's five grid kernels beat ours on zen4 while losing on zen2. + Headline: the reference has NO AVX-512 path for them - a znver4 build runs the same AVX2 body as zen2 (256-bit + vpmaddubsw, never vpdpbusd) and its 1.4-1.9x zen2 -> zen4 gain is the core alone; ours gained 1.14-1.26x because + we are stall-bound (fewer instructions, more cycles): the panel round trip (1024 scattered 4-byte stores per + superblock-group re-read as 64-byte vectors) is the one structure we have and they do not. Ranked: the row form + for iq3xxs on vnni512 (measured; loses at 16 SMT lanes - needs bare metal), `gather="qpanel"` (u64 grid entries + stored whole, est 1.2-1.4x on the iq2 three), the parity-completed sign byte baked at repack (5-8%, free for + iq2xxs/iq3xxs), mask-register signs at width 512 (vptestmb + merge-masked vpsubb), the 256-bit gemv seat (the + harness's own crown now), and the VBMI symbol lattice - the grids are a 3-symbol (iq2) / 8-symbol (iq3) + alphabet, so an entry is 12-16 bits and the decode is vpermi2b plus a few widen/shuffle ops, est 2-3x, needs + avx512vbmi in TUNE_KNOWN_FEATURES. Do not seat hardware gathers (llama.cpp measured them losing on a 7950X) + nor vpdpbssd (zen4 has no AVX-VNNI-INT8). - `kernel_parity_research_vk.md` - llama.cpp's cm2 `mul_mm` against our cm2 tile. Headline: the same kernel design (tile geometry, workgroup, decode-in-load callback, one coopMatMulAdd per k-step, zero barriers in the k loop, the same split-k heuristic) - the gap is not the tile. @@ -489,6 +501,9 @@ The arm classes and x86-avx2 have no same-mr alternative (their grids collapse t x86-vnni512 and x86-amx need the re-mint: two rentals with `aws_bootstrap_mint.sh` (build, TEST, the 1B vehicle's --tune, export, both ladders) - running. The export copies every kernels entry, so the gemv seats ride into the class profiles; the profile adoption merges them as ordinary entries. +zen2 smoke of the new harness path (the 1B vehicle, --tune, 16 lanes): provenance ok / ok / avx2;f16c;fma;sse4.2, +49 kernels, every crown unchanged (maddubs256_mr8), no gemv seat written - correct for a class with no same-layout +rival. The x86-avx2 profile stands. CPU decode (gap 2) - the k-quant decode kernels were the tails the full ladder exposed (k3 0.80x, k6 0.90x, k5 0.91x): From d35f24b1ceda71e4252c29e639e9c460cbd3c1a8 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:54:59 -0700 Subject: [PATCH 069/123] grid row form, x86: DASLLAMA_GRID_QPANEL=1 builds the row group's vector through the idle gather panel - eight qword stores and one load instead of the insert chain The memo's "qpanel" (kernel_parity_research_zen4.md): the row form's cost on x86 is the vmovq/vpinsrq/vinserti chain that concatenates eight row qwords into a 64-byte vector; the same vector comes out of eight 8-byte stores into the alloca the row form already owns (64-aligned) and one width-wide load. A lab knob for the zen4 A/B, x86 only, inert by default. TEST 90/90 with the row form and the knob forced on zen2. LLVM_JIT_CODEGEN_VERSION 0x65. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/dasllama/dasllama_gemm_gen.das | 14 +++++++++++++- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 8c7e8c135c..4a43a154f3 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1718,6 +1718,17 @@ def private concat_rows(var te : TileEmit; var rows : array; l return concat_rows(te, nxt, 2 * lanes) } +//! the row group's weight vector through the gather panel instead of the insert chain: one 8-byte store per row into +//! the (otherwise idle) alloca, one width-wide load - the memo's "qpanel"; x86 only, DASLLAMA_GRID_QPANEL=1 (lab knob) +def private concat_rows_qpanel(var te : TileEmit; var rows : array) : LLVMOpaqueValue? { + let b = te.builder + for (i, r in count(), rows) { + var slot = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, te.types->ConstI64(uint64(i * 8)), "") + LLVMBuildStore(b, LLVMBuildBitCast(b, r, te.types.t_int64, ""), slot) + } + return LLVMBuildLoad2Aligned(b, te.vwi8, te.iq3s_panel, 64u, "wq") +} + //! one row group's dot with its +-1 sign bytes: the signed-weight lattices (sdot, vpdpbssd) multiply them //! into the magnitudes; the unsigned-weight ones (maddubs, vpdpbusd) put them on the activation copy def private grid_rows_dot(var te : TileEmit; var acc, w, sg, xv : LLVMOpaqueValue?) : LLVMOpaqueValue? { @@ -1765,6 +1776,7 @@ def private emit_grid_rows_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue? let b = te.builder let n = te.width / 64 let colSigns = te.kq == 33 || te.kq == 23 || te.dotKind != DOT_SDOT // x86: the sign column's mask beats four table loads + three inserts per group; the sdot lattice keeps the table for the ksigns formats + let qpanel = te.dotKind != DOT_SDOT && te.iq3s_panel != null && get_env_variable("DASLLAMA_GRID_QPANEL") == "1" for (qd in range(te.rq)) { var acc : LLVMOpaqueValue? [2] // per row group; the lo strip (l < 2) when iq2 var acc1 : LLVMOpaqueValue? [2] // iq2: the hi strip (l >= 2) @@ -1787,7 +1799,7 @@ def private emit_grid_rows_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue? sgs |> push(sign_pm1_row(te, kq_row_sign_code(te, base, blk, l))) } } - var w = concat_rows(te, rows) + var w = qpanel ? concat_rows_qpanel(te, rows) : concat_rows(te, rows) var sg = colSigns ? sign_col_pm1(te, signCol[qd], r0, l, "sp{blk}_{l}_{qd}_{g}") : concat_rows(te, sgs) let hi = iq2 && l >= 2 var d = grid_rows_dot(te, hi ? acc1[g] : acc[g], w, sg, xv) diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 17ab94c3f0..edb1f3096f 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x64ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x65ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) From 5869d5b4f94129142dabfaff5d66258a35a2071e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:55:22 -0700 Subject: [PATCH 070/123] tune profiles x86-vnni512 and x86-amx: re-minted on the emitter they ship with (zen4 c7a, Granite Rapids c8i) Both boxes: TEST 103/103, provenance ok / ok. x86-vnni512 (EPYC 9R14): the k3 tile crown moves dot_vpdpbusd_width512_mr16 -> dot_maddubs_width256_mr8 after the k3 plane transpose, and the k3 gemv takes its own same-layout seat dot_vpdpbusd_width256_mr8 - the first entry the gemv's own crown ever wrote; q51's tile moves 512_mr16 -> vpdpbusd_width256_mr8; axpy_q8kv and rope_scaled_neox_tab pick other widths; 50 entries. x86-amx (Xeon 6975P-C): every crown holds, axpy_f16 vec16 -> vec16_u2; 49 entries. The new width-256 / mr16 rows did not clear the margin on either box. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../defaults/x86-amx.tune-defaults.json | 10 +- .../defaults/x86-vnni512.tune-defaults.json | 137 +++++++++--------- 2 files changed, 74 insertions(+), 73 deletions(-) diff --git a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json index 831a423d80..f9b5ac9391 100644 --- a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json @@ -13,7 +13,7 @@ "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "quantize_q8kv_row" : "plain", - "axpy_f16" : "vec16", + "axpy_f16" : "vec16_u2", "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "axpy_tq4kv" : "vec8_u2", "cvt_tq4kv_to_f32" : "vec8_u2", @@ -52,18 +52,18 @@ }, "provenance" : { "validation" : "ok", - "noise_probes" : "start cv 0.20%; mid1 cv 0.06%; mid2 cv 0.09%; end cv 0.19%", + "noise_probes" : "start cv 0.08%; mid1 cv 0.05%; mid2 cv 0.14%; end cv 0.05%", "platform" : "linux", - "noise_floor_cv_pct" : "0.20", + "noise_floor_cv_pct" : "0.14", "features" : "avx2;f16c;fma;sse4.2;avxvnni;avx512f;avx512bw;avx512vl;avx512vnni;amx-tile;amx-int8", "class" : "x86-amx", - "written" : "2026-09-01T11:02:47.521Z", + "written" : "2026-09-01T17:51:26.103Z", "validation_demoted" : "0", "mode" : "normal", "dasllama_version" : "15", "origin" : "profile", "noise" : "ok", "arch" : "x86_64", - "validation_max_drift_pct" : "1.97" + "validation_max_drift_pct" : "4.97" } } \ No newline at end of file diff --git a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json index ebedf6abd3..7c17a09fd7 100644 --- a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json @@ -1,69 +1,70 @@ { - "kernels": { - "add_inplace": "vec8_u2", - "add_scale_inplace": "vec8_u2", - "axpy": "vec8_u2", - "axpy_f16": "vec8_u2", - "axpy_q8kv": "plain", - "axpy_tq4kv": "vec8_u2", - "copy_floats": "vec8_u2", - "cvt_f16_to_f32": "vec8_u2", - "cvt_f32_to_f16": "plain", - "cvt_q8kv_to_f32": "vec8_u2", - "cvt_tq4kv_to_f32": "vec8_u2", - "dot": "vec16", - "dot_bf16": "vec8_u2", - "dot_f16": "vec16", - "dot_mx4q8": "u2", - "dot_q4": "vec8_u2", - "dot_q51e": "vec16", - "dot_q8kv": "vec8_u2", - "dot_q8q8": "u2", - "dot_q8q8_f16s": "vec16", - "dot_q8q8_laneq4x4": "", - "dot_q8q8kv": "u2", - "dot_q8tq4kv": "vec16", - "gemm_f32_uk_4x16": "u2", - "iq2sq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq2xsq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq2xxsq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq3sq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq3xxsq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq4nlq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "iq4xsq8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k2q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k3q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k4q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k5q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "k6q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "mul_inplace": "vec8_u2", - "q40q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "q51q8_tile_gen": "dot_vpdpbusd_width512_mr16", - "q8q8_tile_gen": "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", - "quantize_q8_0_bs_into_ptr": "u2", - "quantize_q8_0_into_ptr": "plain", - "quantize_q8kv_row": "plain", - "quantize_tq4kv_row": "plain", - "rmsnorm": "vec8", - "rope_scaled_neox_tab": "vec8_u2", - "scale_inplace": "vec8_u2", - "softmax": "vec8_u2", - "softmax_sink": "vec8_u2" - }, - "provenance": { - "arch": "x86_64", - "class": "x86-vnni512", - "dasllama_version": "15", - "features": "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni", - "mode": "normal", - "noise": "ok", - "noise_floor_cv_pct": "0.21", - "noise_probes": "start cv 0.09%; mid1 cv 0.04%; mid2 cv 0.21%; end cv 0.06%", - "origin": "profile", - "platform": "linux", - "validation": "ok", - "validation_demoted": "3", - "validation_max_drift_pct": "21.77", - "written": "2026-09-01T01:30:04.021Z" - } -} + "kernels" : { + "add_inplace" : "vec8_u2", + "cvt_f32_to_f16" : "plain", + "rope_scaled_neox_tab" : "plain", + "q51q8_tile_gen" : "dot_vpdpbusd_width256_mr8", + "softmax" : "vec8_u2", + "mul_inplace" : "vec8_u2", + "quantize_q8_0_bs_into_ptr" : "u2", + "iq2xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8kv" : "vec8_u2", + "dot_q8q8" : "u2", + "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "quantize_q8kv_row" : "plain", + "axpy_f16" : "vec8_u2", + "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "axpy_tq4kv" : "vec8_u2", + "cvt_tq4kv_to_f32" : "vec8_u2", + "axpy" : "vec8_u2", + "dot_q8q8kv" : "u2", + "dot_mx4q8" : "u2", + "softmax_sink" : "vec8_u2", + "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8q8_laneq4x4" : "", + "dot_bf16" : "vec8_u2", + "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "add_scale_inplace" : "vec8_u2", + "cvt_q8kv_to_f32" : "vec8_u2", + "axpy_q8kv" : "vec8_u2", + "dot_q8q8_f16s" : "vec16", + "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", + "k3q8_gemv_gen" : "dot_vpdpbusd_width256_mr8", + "k4q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k3q8_tile_gen" : "dot_maddubs_width256_mr8", + "k2q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "quantize_q8_0_into_ptr" : "plain", + "gemm_f32_uk_4x16" : "u2", + "dot_q51e" : "vec16", + "dot_f16" : "vec16", + "cvt_f16_to_f32" : "vec8_u2", + "dot" : "vec16", + "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "dot_q8tq4kv" : "vec16", + "scale_inplace" : "vec8_u2", + "dot_q4" : "vec8_u2", + "quantize_tq4kv_row" : "plain", + "copy_floats" : "vec8_u2", + "rmsnorm" : "vec8" + }, + "provenance" : { + "validation" : "ok", + "noise_probes" : "start cv 0.15%; mid1 cv 0.08%; mid2 cv 0.18%; end cv 0.08%", + "platform" : "linux", + "noise_floor_cv_pct" : "0.18", + "features" : "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni", + "class" : "x86-vnni512", + "written" : "2026-09-01T17:52:25.911Z", + "validation_demoted" : "1", + "mode" : "normal", + "dasllama_version" : "15", + "origin" : "profile", + "noise" : "ok", + "arch" : "x86_64", + "validation_max_drift_pct" : "6.18" + } +} \ No newline at end of file From 9ace1f89d32808c36cc733dc58ea46d615ecfcc0 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 10:59:42 -0700 Subject: [PATCH 071/123] gemm emitter: the ymm vreg budget counts 32 registers on EVEX targets perm_declines budgeted 16 ymm registers at width 256, so every width-256 row with two weight vectors per group (mr 16) declined into the reference body - the re-mint's dot_vpdpbusd_width256_mr16 rows timed exactly like the reference on both boxes (3168491 vs 3167365 us on zen4), which the previous commit misread as "did not clear the margin". A row that requires AVX-512 runs where EVEX encodes ymm16-31, so the budget is 32 there. LLVM_JIT_CODEGEN_VERSION 0x66. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/dasllama/dasllama_gemm_gen.das | 4 ++-- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 4a43a154f3..117da8ae1c 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -226,8 +226,8 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint let vq = p.mr / rv if ((vq > 2) // the weight bank is 8 dword-groups x vq || (p.nrsplit * vq > 8)) return true // accumulator bank (emit_slice phi arrays) - // vreg budget: float + int acc tiles + x splat/sign scratch + weight/|weight| in flight - return 2 * p.nrsplit * vq + 6 > (p.width == 512 ? 32 : 16) + // vreg budget: float + int acc tiles + x splat/sign scratch + weight/|weight| in flight; EVEX targets have 32 ymm too + return 2 * p.nrsplit * vq + 6 > (p.width == 512 || g_target_x64_avx512bw ? 32 : 16) } // emission context threaded through the helpers: builder/types + the per-function value handles diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index edb1f3096f..b4fffc9edd 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x65ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x66ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) From 3a8ec2e0d621da7826c4d43c5be4e00f4b2d4d77 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:00:35 -0700 Subject: [PATCH 072/123] kernel parity plan: the re-mint round, the declined width-256 row, the zen2 qpanel A/B --- plans/kernel_parity_pass.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 9d32becb09..002f456712 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -752,5 +752,24 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. - 2026-09-01: sign=vec for iq3xxs/iq2xs/iq2xxs via the parity-synthesized column (no layout change) and the u64 grid pair load for the three iq2 formats: iq2s 5152 us (0.98x), iq2xxs 5209 (0.98x), iq2xs 6297 (0.86x), iq3xxs 7893 (0.83x), iq3s 7562 (1.37x); 85 variants ok in TEST mode. +- 2026-09-01: re-mint round on the shipping emitter (commit 11f69493e). zen4 c7a (x86-vnni512, 50 + entries, TEST 103/103): the k3 tile crown moved 512_mr16 -> maddubs_width256_mr8 (tile race 51347 + vs 52840 us; the transposed k3 planes made the 256 maddubs seat the tile winner) and the k3 gemv + took its own same-layout seat vpdpbusd_width256_mr8 (374 vs the tile seat's 401 us) - the first + entry the gemv's own crown wrote; q51 tile 512_mr16 -> vpdpbusd_width256_mr8; axpy_q8kv plain -> + vec8_u2, rope_scaled_neox_tab vec8_u2 -> plain. Intel c8i (x86-amx, 49 entries, TEST 103/103): + every crown holds (k3 41768 vs 57342, k6 33760 vs 53832 for the 512 tile), axpy_f16 vec16 -> + vec16_u2, no gemv seat - so the Intel k6 normal-mode 5.2 ms gap stays open. +- 2026-09-01: the new dot_vpdpbusd_width256_mr16 row was a dud on both boxes - its tile time equalled + the reference row's (zen4 3168491 vs 3167365 us): perm_declines budgeted 16 ymm registers at width + 256, so a two-vectors-per-group row (nrsplit 4: 2*4*2+6 = 22) declined into the reference body. A + row that requires AVX-512 runs where EVEX encodes ymm16-31; the budget is 32 there (461edda72, + LLVM_JIT_CODEGEN_VERSION 0x66). The row is un-raced until the next mint. +- 2026-09-01: the qpanel spelling (DASLLAMA_GRID_QPANEL=1: eight qword stores into the alloca panel + + one width load instead of the insert chain, commit eec5ce80c) measured on zen2 with the row form + forced, one thread at the engine phase, qpanel 0 -> 1: iq2xs 8797 -> 8776, iq2s 6733 -> 7435, iq3s + 11886 -> 11647, iq3xxs 7212 -> 7262, iq2xxs 4624 -> 5841 us. A loss or a wash on zen2 (where the row + form itself loses to the panel form: iq2xs 4565 panel); the zen4 A/B is the one that matters + (iq2xxs's crown there IS the row form) - queued behind the box's ladder. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From c8c1ea610bec77fc3842e3c8b581b4fc4ca1f19e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:07:29 -0700 Subject: [PATCH 073/123] kernel parity plan: zen4 v3 and Intel v3 ladders on the re-minted profiles --- plans/kernel_parity_pass.md | 52 +++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 002f456712..48b8eda3d2 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -441,6 +441,58 @@ q40 0.95, k5 0.94, iq4xs 0.93, q8 0.92 - and the grid formats iq3xxs 0.90, iq2xs the crowns; iq2s 2528 / 2383 / 2530 - the crowned width-512 seat loses 6% to vpdpbusd256 at the engine shape, the gemv inheriting the tile's crown (tune_companion). perf stat gave nothing (no PMU in the VM). +### zen4 v3 - the re-minted x86-vnni512 profile (2026-09-01, c7a.4xlarge EPYC 9R14, commit 11f69493e; `SOLO=1 NTOK=512` one thread at d=4096 for gemv/tile, then the team ladder: 16 lanes, d=32768, best of 5) + +| fmt | gemv 1T ours / ref | ratio | tile 1T ours / ref | ratio | gemv 16 lanes ours / ref | ratio | +|---|---|---|---|---|---|---| +| q8 | 1983 / 3650 | 1.84 | 167509 / 501447 | 2.99 | 6671 / 5995 | 0.90 | +| k4 | 843 / 1878 | 2.23 | 350323 / 753000 | 2.15 | 2914 / 3049 | 1.05 | +| k5 | 2160 / 2685 | 1.24 | 178164 / 1124541 | 6.31 | 3873 / 3824 | 0.99 | +| k6 | 2478 / 2992 | 1.21 | 226354 / 787084 | 3.48 | 4637 / 4682 | 1.01 | +| q40 | 693 / 2726 | 3.93 | 173654 / 588905 | 3.39 | 3027 / 2940 | 0.97 | +| q51 | 1559 / 4640 | 2.98 | 285388 / 1428594 | 5.01 | 4327 / 4300 | 0.99 | +| iq4xs | 1129 / 2449 | 2.17 | 286750 / 1155395 | 4.03 | 3225 / 2856 | 0.89 | +| k3 | 942 / 2022 | 2.15 | 355086 / 979214 | 2.76 | 2212 / 2161 | 0.98 | +| iq3s | 7556 / 6092 | 0.81 | 261572 / 3093452 | 11.83 | 3896 / 3260 | 0.84 | +| iq3xxs | 5964 / 5226 | 0.88 | 260290 / 2660673 | 10.22 | 3176 / 2818 | 0.89 | +| iq4nl | 1075 / 3083 | 2.87 | 284054 / 624404 | 2.20 | 3115 / 2882 | 0.93 | +| k2 | 622 / 1281 | 2.06 | 430224 / 641204 | 1.49 | 1433 / 1611 | 1.12 | +| iq2s | 4915 / 3756 | 0.76 | 283819 / 1903444 | 6.71 | 2524 / 2124 | 0.84 | +| iq2xs | 4555 / 3621 | 0.79 | 283489 / 1845848 | 6.51 | 2559 / 2003 | 0.78 | +| iq2xxs | 2854 / 3916 | 1.37 | 260753 / 1992837 | 7.64 | 1657 / 2116 | 1.28 | +| mx4 | 1059 / 2746 | 2.59 | 292052 / 1349795 | 4.62 | 2895 / 2864 | 0.99 | + +Reading: one thread, everything is past the reference except the four panel-form grids (iq3s 0.81, iq3xxs +0.88, iq2s 0.76, iq2xs 0.79; iq2xxs on the row form 1.37) - the compute-bound gap the zen4 memo targets. At +16 lanes the box is DRAM-bound (k4 ours 235 MB / 2.9 ms = 81 GB/s, the reference 77): the 0.89-0.99 rows +there are within the shared box's bandwidth noise, the grids repeat their one-thread losses (0.78-0.89). + +### Intel v3 - the re-minted x86-amx profile (2026-09-01, c8i.4xlarge Xeon 6975P-C, commit 11f69493e; same ladders) + +| fmt | gemv 1T ours / ref | ratio | tile 1T ours / ref | ratio | gemv 16 lanes ours / ref | ratio | +|---|---|---|---|---|---|---| +| q8 | 2226 / 2813 | 1.26 | 153424 / 578235 | 3.77 | 4414 / 4319 | 0.98 | +| k4 | 1138 / 1704 | 1.50 | 234490 / 847640 | 3.61 | 2057 / 3083 | 1.50 | +| k5 | 1567 / 2471 | 1.58 | 176873 / 1185485 | 6.70 | 2807 / 4063 | 1.45 | +| k6 | 1831 / 1883 | 1.03 | 218123 / 912830 | 4.18 | 5227 / 4353 | 0.83 | +| q40 | 1107 / 2289 | 2.07 | 168018 / 649524 | 3.87 | 2065 / 5143 | 2.49 | +| q51 | 1472 / 2915 | 1.98 | 227348 / 1462531 | 6.43 | 2907 / 5406 | 1.86 | +| iq4xs | 1109 / 2378 | 2.14 | 280699 / 1158466 | 4.13 | 2169 / 4817 | 2.22 | +| k3 | 1272 / 2269 | 1.78 | 297557 / 1143046 | 3.84 | 2642 / 2687 | 1.02 | +| iq3s | 5152 / 7079 | 1.37 | 251684 / 3550699 | 14.11 | 7426 / 7413 | 1.00 | +| iq3xxs | 4386 / 5073 | 1.16 | 250846 / 2597215 | 10.35 | 4533 / 6704 | 1.48 | +| iq4nl | 1113 / 2710 | 2.44 | 264126 / 714114 | 2.70 | 2079 / 6320 | 3.04 | +| k2 | 660 / 1470 | 2.23 | 287013 / 699910 | 2.44 | 957 / 1729 | 1.81 | +| iq2s | 3963 / 3690 | 0.93 | 275657 / 1885050 | 6.84 | 4266 / 3848 | 0.90 | +| iq2xs | 3914 / 4326 | 1.11 | 275710 / 2206824 | 8.00 | 3931 / 4610 | 1.17 | +| iq2xxs | 2505 / 3666 | 1.46 | 251397 / 1828919 | 7.28 | 2589 / 4255 | 1.64 | +| mx4 | 1207 / 3038 | 2.52 | 275059 / 1311864 | 4.77 | 2488 / 6361 | 2.56 | + +Reading: 16 lanes at or past the reference except k6 0.83 (5227 us - the normal-mode gap, see the queue; +one thread shows it as best 1831 / median 2925 inside ONE process, so the slow mode is the common one and +flips between rounds, not between processes) and iq2s 0.90 (0.93 one thread). No gemv seat was written on +this box, so the k6 gemv still rides the 512 tile crown here. + ## 3. Research memos (read before touching the kernels) - `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, From 611bd35029cb9ea6d04d50e3671b4ac2d17d0abf Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:11:52 -0700 Subject: [PATCH 074/123] kq_kernel_bench: --each appends every round's sample - the per-round view of a bimodal row --- modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index e0265e8ad3..818c15be74 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -53,6 +53,8 @@ struct BenchArgs { base_offset : int = 0 @clarg_doc = "GEMV the engine's way: the engine's own splitter (matmul_chunks_gemv) and dispatcher (maybe_parallel_for_indexed) over DAS_JOBQUE_THREADS lanes, wall = the slowest lane, the lanes' fastest and slowest summed call time in two extra columns; 0 = one raw call on this thread" team : bool + @clarg_doc = "Append every round's sample (us, in round order) as one more column - the per-round view of a bimodal row" + each : bool } typedef GemvFn = function<(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void> @@ -252,6 +254,7 @@ struct RowStat { } let TEAM_BURN = 6 // warmup dispatches per row before the team arm times anything +var g_each_round = false // --each //! the engine's decode dispatch, verbatim (dasllama_math's matmul shape): the splitter's chunk count over the //! rows, the team self-serving them; each lane adds its call time to laneP[slot] @@ -337,7 +340,11 @@ def emit_row(var rows : int&; name, tier : string; var st : RowStat; n, d, ntok, let mode = get_env_variable("DAS_TUNE_MODE") let perm = (st.perm == "reference" && mode != "tune" && mode != "test") ? "stamped" : st.perm // normal mode: the registry's one row IS the stamped winner let lanes = st.lane_hi > 0l ? "\t{st.lane_lo}\t{st.lane_hi}" : "" - print("{name}\t{tier}\t{perm}\t{n}\t{d}\t{ntok}\t{best}\t{med}\t{ns_per_sb}\t{gbs}\t{gflops}{lanes}\n") + var each = "" + if (g_each_round) { + each = "\t" + join(st.us, ",") + } + print("{name}\t{tier}\t{perm}\t{n}\t{d}\t{ntok}\t{best}\t{med}\t{ns_per_sb}\t{gbs}\t{gflops}{lanes}{each}\n") rows++ } @@ -630,6 +637,7 @@ def main : int { let rc = parse_args_with_help(cfg, "kq_kernel_bench") BASE_ALIGN = int64(cfg.base_align) BASE_OFFSET = int64(cfg.base_offset) + g_each_round = cfg.each return rc if (rc >= 0) if (!jit_enabled()) { to_log(LOG_ERROR, "run with -jit: the kq generators only fire under the JIT\n") From 6bd09195e9d0c426c6d3d3741f9fd45c0b70d878 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:16:27 -0700 Subject: [PATCH 075/123] grid row form: the qpanel spelling is gone - a loss or a wash on zen2, zen4 and Intel Row form forced, one thread, d=32768, qpanel 0 -> 1: zen4 iq2xs 33680 -> 40895, iq2s 47037 -> 44739, iq3s 68550 -> 70264, iq3xxs 43468 -> 46495, iq2xxs 22687 -> 23111 us; Intel iq2s 44660 -> 56802, the rest within noise; zen2 iq2xxs 4624 -> 5841. The insert chain stays. TEST 103/103 on zen2. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/dasllama/dasllama_gemm_gen.das | 14 +------------- plans/kernel_parity_pass.md | 7 +++++++ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 117da8ae1c..fa825a4314 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1718,17 +1718,6 @@ def private concat_rows(var te : TileEmit; var rows : array; l return concat_rows(te, nxt, 2 * lanes) } -//! the row group's weight vector through the gather panel instead of the insert chain: one 8-byte store per row into -//! the (otherwise idle) alloca, one width-wide load - the memo's "qpanel"; x86 only, DASLLAMA_GRID_QPANEL=1 (lab knob) -def private concat_rows_qpanel(var te : TileEmit; var rows : array) : LLVMOpaqueValue? { - let b = te.builder - for (i, r in count(), rows) { - var slot = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, te.types->ConstI64(uint64(i * 8)), "") - LLVMBuildStore(b, LLVMBuildBitCast(b, r, te.types.t_int64, ""), slot) - } - return LLVMBuildLoad2Aligned(b, te.vwi8, te.iq3s_panel, 64u, "wq") -} - //! one row group's dot with its +-1 sign bytes: the signed-weight lattices (sdot, vpdpbssd) multiply them //! into the magnitudes; the unsigned-weight ones (maddubs, vpdpbusd) put them on the activation copy def private grid_rows_dot(var te : TileEmit; var acc, w, sg, xv : LLVMOpaqueValue?) : LLVMOpaqueValue? { @@ -1776,7 +1765,6 @@ def private emit_grid_rows_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue? let b = te.builder let n = te.width / 64 let colSigns = te.kq == 33 || te.kq == 23 || te.dotKind != DOT_SDOT // x86: the sign column's mask beats four table loads + three inserts per group; the sdot lattice keeps the table for the ksigns formats - let qpanel = te.dotKind != DOT_SDOT && te.iq3s_panel != null && get_env_variable("DASLLAMA_GRID_QPANEL") == "1" for (qd in range(te.rq)) { var acc : LLVMOpaqueValue? [2] // per row group; the lo strip (l < 2) when iq2 var acc1 : LLVMOpaqueValue? [2] // iq2: the hi strip (l >= 2) @@ -1799,7 +1787,7 @@ def private emit_grid_rows_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue? sgs |> push(sign_pm1_row(te, kq_row_sign_code(te, base, blk, l))) } } - var w = qpanel ? concat_rows_qpanel(te, rows) : concat_rows(te, rows) + var w = concat_rows(te, rows) var sg = colSigns ? sign_col_pm1(te, signCol[qd], r0, l, "sp{blk}_{l}_{qd}_{g}") : concat_rows(te, sgs) let hi = iq2 && l >= 2 var d = grid_rows_dot(te, hi ? acc1[g] : acc[g], w, sg, xv) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 48b8eda3d2..660e21d2ae 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -823,5 +823,12 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. 11886 -> 11647, iq3xxs 7212 -> 7262, iq2xxs 4624 -> 5841 us. A loss or a wash on zen2 (where the row form itself loses to the panel form: iq2xs 4565 panel); the zen4 A/B is the one that matters (iq2xxs's crown there IS the row form) - queued behind the box's ladder. +- 2026-09-01: qpanel killed - the zen4 and Intel A/Bs (row form forced, one thread, d=32768, qpanel 0 -> 1): + zen4 iq2xs 33680 -> 40895, iq2s 47037 -> 44739, iq3s 68550 -> 70264, iq3xxs 43468 -> 46495, iq2xxs 22687 -> + 23111 us; Intel iq2xs 44067 -> 43923, iq2s 44660 -> 56802, iq3s 84893 -> 86998, iq3xxs 39726 -> 39662, iq2xxs + 22041 -> 21450. A loss or a wash on all three boxes; the helper and the knob are gone from the emitter. + Same run, a find: on zen4 the row form beats the panel form for iq2xs at 16 lanes (2194 vs the ladder's panel + 2559 us - 0.91x of the reference instead of 0.78x) where the class gate picks the panel; the forms probe + (zen4_forms.sh, both forms x five grids x two shapes) decides the gate. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 3373beae3f87347980b77829e4918620bd1ea186 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:27:33 -0700 Subject: [PATCH 076/123] grid gemv: the VBMI symbol lattice as a perm row for iq2xxs and iq3xxs (grid = "vbmi"; lab, unraced) The memo's item 6. A grid entry is a word over a tiny alphabet (iq2xxs 3 symbols, iq3xxs 8), so the whole decode runs in vectors: the block's index column (16 rows x 4 bytes, one load) looks its compact codes up through two VPERMI2B per plane plus a blend on bit 7; the aux column's four 7-bit codes per row become sign bytes through one ksigns VPERMI2B (parity included, 128 bytes = the two registers it indexes); per row group and weight octet a constant two-source shuffle places each row's code bytes in its qword, VPMULTISHIFTQB spreads the symbols into bytes, one vpshufb maps them to magnitudes, and the signs ride the activation copy as a mask (x ^ m) - m. Same tile and planes as the 512/mr16 row - only the gemv differs, which is what the gemv's own seat races. Plumbing: g_target_x64_avx512vbmi (cpuid leaf 7 ecx bit 1 in das_cpu_supports - a binary rebuild), avx512vbmi in TUNE_KNOWN_FEATURES, TilePerm.grid parsed and gated on every leg (zmm VBMI only), LLVM_JIT_CODEGEN_VERSION 0x67. Compiles and declines on zen2; TEST and the race on the VBMI boxes follow. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 235 +++++++++++++++++- .../dasLLAMA/dasllama/dasllama_math_gen.das | 2 + modules/dasLLVM/daslib/llvm_jit_common.das | 2 + modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- modules/dasLLVM/daslib/llvm_tune.das | 2 +- src/builtin/module_builtin_runtime.cpp | 1 + 6 files changed, 238 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index fa825a4314..2c19ed453f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -98,6 +98,8 @@ struct private TilePerm { // scratch, block i−1's scale fold interleaved with block i's TMUL ops latch : int = 0 // 1 = raw immediate-tmm amx ops (slice K I1): no per-call // ldtilecfg/tilerelease — the cfg companion owns per-chunk config + grid : string = "" // "vbmi": the grid formats' gemv decodes through the VBMI symbol lattice + // (the row form; the tile and the planes are the plain row's) } def private parse_perm(gc : LlvmCodeCtx) : TilePerm { @@ -120,6 +122,8 @@ def private parse_perm(gc : LlvmCodeCtx) : TilePerm { if (pv is tInt) { p.pipe = pv as tInt } let lv = find_arg(gc.ann.arguments, "latch") if (lv is tInt) { p.latch = lv as tInt } + let grv = find_arg(gc.ann.arguments, "grid") + if (grv is tString) { p.grid = grv as tString } return p } @@ -136,6 +140,7 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint || (p.bias != 0 && p.bias != 128) || (p.pipe != 0 && p.pipe != 1) || (p.latch != 0 && p.latch != 1) + || (p.grid != "") || (!g_target_is_x64) || (!g_target_x64_amx) || (!g_target_x64_avx512vnni) // the companions' busd512 lattice + zmm fold || (!g_target_os_linux)) return true @@ -171,12 +176,13 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint || (p.nrsplit != 2 && p.nrsplit != 4) || (p.gkstep != 1 && p.gkstep != 2 && p.gkstep != 4) || (p.mr < 4) || (p.pipe != 0) || (p.latch != 0) + || (p.grid != "" && p.grid != "vbmi") || (p.bias != 0 && (p.bias != 128 || p.dotPrim != "vpdpbusd"))) return true let rt = q8q8_repack_type(p.mr, p.bias, kgroup_of(p)) if (rt.interleave != p.mr || !rt.f32_scales) return true if (p.dotPrim == "smmla") { // the i8mm MMA leg: NEON widths, row-quad geometry, same q-reg budget as sdot, and the i8mm tier flag (M1 is dotprod-only) - if ((!g_target_is_aarch64) || (!g_target_arm64_i8mm) || (p.width != 128) + if ((!g_target_is_aarch64) || (!g_target_arm64_i8mm) || (p.width != 128) || (p.grid != "") || (p.mr % 4 != 0) || (2 * p.mr + p.nrsplit * p.mr / 4 + 2 * p.nrsplit + 2 > 32)) return true let mod = LLVMGetGlobalParent(gc.impl) @@ -194,7 +200,7 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint } if (p.dotPrim == "sdot") { // the NEON leg (M2 shape): 128-bit q-regs, 4-row sdot lanes; mr>=8 budgets the FUSED-acc shape (kq shares one lo+hi acc there — half the mr4 formula's int-acc bank) - if ((!g_target_is_aarch64) || (p.width != 128) + if ((!g_target_is_aarch64) || (p.width != 128) || (p.grid != "") || (p.mr % 4 != 0) || (p.mr >= 8 ? (p.mr + p.nrsplit * p.mr / 4 + 2 * p.nrsplit + 2 > 32) : (2 * p.mr + p.nrsplit * p.mr / 4 + 2 * p.nrsplit + 2 > 32))) return true @@ -213,6 +219,7 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint || (!g_target_is_x64) || (p.width != 256 && p.width != 512) || (p.dotPrim == "vpdpbssd" && p.width != 256)) return true // 512-bit bssd is AVX10 — unproven + if (p.grid == "vbmi" && (p.width != 512 || p.dotPrim == "vpdpbssd" || !g_target_x64_avx512vbmi)) return true // the lattice is zmm VBMI if (p.width == 256) { if ((p.dotPrim == "maddubs" && !g_target_x64_avx2) || (p.dotPrim == "vpdpbusd" && !g_target_x64_vnni256) @@ -251,6 +258,16 @@ struct private TileEmit { iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) sign_code_pm1_g : LLVMOpaqueValue? // the row-group grid decode: ksigns code -> eight +-1 bytes, [128 x i64], parity bit folded in gridRows : bool // set while the row-group grid decode emits: the column dword read (a panel-form win) reverts to byte loads + vbmi : bool // grid = "vbmi": the grid gemv decodes through the VBMI symbol lattice (row form) + vb_plane : LLVMOpaqueValue? [2] // the grid's compact code planes, [256 x i8] each: the entry's low / high symbols + vb_alpha : LLVMOpaqueValue? // the symbol alphabet as a vpshufb table (16 bytes per lane, [64 x i8]) + vb_ksigns : LLVMOpaqueValue? // ksigns_iq2xs as [128 x i8] - one VPERMI2B turns 7-bit codes into sign bytes, parity in + permi2b_decl : LLVMOpaqueValue? // llvm.x86.avx512.vpermi2var.qi.512 + permi2b_ty : LLVMOpaqueType? + mshift_decl : LLVMOpaqueValue? // llvm.x86.avx512.pmultishift.qb.512 + mshift_ty : LLVMOpaqueType? + shufb_decl : LLVMOpaqueValue? // llvm.x86.avx512.pshuf.b.512 (the lattice's alphabet lookup) + shufb_ty : LLVMOpaqueType? gfn : LLVMOpaqueValue? // the function under emit (block creation for emitted inner loops) gctx : LLVMOpaqueContext? psign_ty : LLVMOpaqueType? @@ -1467,6 +1484,7 @@ def private sign_col_pm1(var te : TileEmit; var col : LLVMOpaqueValue?; r0, l : //! scale with the core, so x86-vnni512 takes the row form for iq2s and iq2xxs (zen4: 4918 -> 3600, 4866 -> 4300 us) //! and the panel elsewhere (iq2xs / iq3xxs lose 1.8x, zen2 loses on all five); DASLLAMA_GRID_ROWS_X86=1 / 0 forces it (clear .jitted_scripts between runs) def private grid_rows_path(te : TileEmit) : bool { + if (te.vbmi) return true if (te.dotKind == DOT_SDOT) return te.rv == 4 if (te.dotKind != DOT_MADDUBS && te.dotKind != DOT_VPDPBUSD && te.dotKind != DOT_VPDPBSSD) return false let knob = get_env_variable("DASLLAMA_GRID_ROWS_X86") @@ -1758,6 +1776,199 @@ def private row_weights8(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l return LLVMBuildBitCast(b, v, v8i8, "") } +//! the lattice's constant tables, emitted once per module: the grid's two compact code planes (entry e's +//! low / high symbols, 2 bits each for the iq2 alphabet, 3 for iq3), the alphabet as a per-lane vpshufb +//! table, and ksigns_iq2xs whole (128 bytes: exactly the two registers one VPERMI2B indexes) +def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { + let tag = te.kq == 25 ? "iq2xxs" : "iq3xxs" + te.vb_plane[0] = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.p0") + te.vb_plane[1] = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.p1") + te.vb_alpha = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.alpha") + te.vb_ksigns = LLVMGetNamedGlobal(mod, "dasllama.vbmi.ksigns") + if (te.vb_plane[0] != null && te.vb_plane[1] != null && te.vb_alpha != null && te.vb_ksigns != null) return + var alpha : array + if (te.kq == 25) { + alpha <- [8, 25, 43] + } else { + alpha <- [2, 6, 10, 14, 18, 22, 26, 31] + } + let bits = te.kq == 25 ? 2 : 3 + var p0 : array + var p1 : array + p0 |> reserve(256) + p1 |> reserve(256) + if (te.kq == 25) { + let g = iq2xxs_grid2() + for (e in range(256)) { + p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e], alpha, bits)), 0)) + p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e + 1], alpha, bits)), 0)) + } + } else { + let g = iq3xxs_grid() + for (e in range(256)) { + p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e], alpha, bits) & 255), 0)) + p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e] >> 16u, alpha, bits) & 255), 0)) + } + } + te.vb_plane[0] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p0", p0) + te.vb_plane[1] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p1", p1) + var al : array + al |> reserve(64) + for (k in range(64)) { + al |> push(LLVMConstInt(te.types.t_int8, uint64(k % 16 < length(alpha) ? alpha[k % 16] : 0), 0)) + } + te.vb_alpha = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.alpha", al) + let ks = ksigns_iq2xs() + var kv : array + kv |> reserve(128) + for (k in range(128)) { + kv |> push(LLVMConstInt(te.types.t_int8, uint64(ks[k]), 0)) + } + te.vb_ksigns = vbmi_const_bytes(te, mod, "dasllama.vbmi.ksigns", kv) +} + +//! a grid word's four weight bytes as packed symbols (weight t at bits t*bits); a byte outside the alphabet is a broken table +def private vbmi_pack_word(word : uint; alpha : array; bits : int) : int { + var code = 0 + for (t in range(4)) { + let wb = int((word >> uint(8 * t)) & 0xFFu) + let si = alpha |> find_index(wb) + if (si < 0) { + panic("vbmi lattice: grid byte {wb} is outside the alphabet") + } + code |= si << (bits * t) + } + return code +} + +def private vbmi_const_bytes(var te : TileEmit; mod : LLVMOpaqueModule?; name : string; var elems : array) : LLVMOpaqueValue? { + var g = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int8, uint(length(elems))), name) + LLVMSetInitializer(g, LLVMConstArray(te.types.t_int8, array_data_ptr(elems), uint(length(elems)))) + LLVMSetGlobalConstant(g, 1) + LLVMSetLinkage(g, LLVMLinkage.LLVMPrivateLinkage) + LLVMSetAlignment(g, 64u) + return g +} + +//! one 64-byte slice of a constant byte table (the lattice's register operands; loop-invariant, LLVM hoists them) +def private vbmi_table_part(var te : TileEmit; var tbl : LLVMOpaqueValue?; part : int; name : string) : LLVMOpaqueValue? { + let b = te.builder + var p = LLVMBuildGEP2(b, te.types.t_int8, tbl, te.types->ConstI64(uint64(part * 64)), "") + return LLVMBuildLoad2Aligned(b, te.vwi8, p, 64u, name) +} + +def private vbmi_permi2b(var te : TileEmit; var lo, idx, hi : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { + var args <- [lo, idx, hi] + return LLVMBuildCall2(te.builder, te.permi2b_ty, te.permi2b_decl, args, name) +} + +//! VPMULTISHIFTQB: byte j of each qword = the 8 bits of the source qword starting at bit ctrl[j] +def private vbmi_mshift(var te : TileEmit; var v : LLVMOpaqueValue?; ctrl : array; name : string) : LLVMOpaqueValue? { + var cs : array + cs |> reserve(64) + for (k in range(64)) { + cs |> push(LLVMConstInt(te.types.t_int8, uint64(ctrl[k % 8]), 0)) + } + var args <- [LLVMConstVector(array_data_ptr(cs), 64u), v] + return LLVMBuildCall2(te.builder, te.mshift_ty, te.mshift_decl, args, name) +} + +//! 64 byte indices through a 256-entry byte table: two VPERMI2B (128 entries each) and one blend on index bit 7 +def private vbmi_lookup256(var te : TileEmit; var tbl, idx : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { + let b = te.builder + var t0 = vbmi_permi2b(te, vbmi_table_part(te, tbl, 0, "{name}.t0"), idx, vbmi_table_part(te, tbl, 1, "{name}.t1"), "{name}.lo") + var t1 = vbmi_permi2b(te, vbmi_table_part(te, tbl, 2, "{name}.t2"), idx, vbmi_table_part(te, tbl, 3, "{name}.t3"), "{name}.hi") + var high = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, idx, LLVMConstNull(te.vwi8), "{name}.b7") + return LLVMBuildSelect(b, high, t1, t0, name) +} + +//! the unsigned-weight dot with the signs as a byte mask on the activation copy: x' = (x ^ m) - m +def private rows_dot_masked(var te : TileEmit; var acc, w, m, xv : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let b = te.builder + var xs = LLVMBuildSub(b, LLVMBuildXor(b, xv, m, ""), m, "signx") + if (te.dotKind == DOT_VPDPBUSD) { + var dargs <- [acc, w, xs] + return LLVMBuildCall2(b, te.dp_ty, te.dp_decl, dargs, "dot") + } + var margs <- [w, xs] + var pairs = LLVMBuildCall2(b, te.madd_ty, te.madd_decl, margs, "pairs") + var wargs <- [pairs, ones_i16(te)] + return LLVMBuildAdd(b, acc, LLVMBuildCall2(b, te.maddwd_ty, te.maddwd_decl, wargs, "quads"), "dot") +} + +//! One block of iq2xxs / iq3xxs as ROW GROUPS through the VBMI symbol lattice: the block's index column +//! (16 rows x 4 bytes) looks its compact codes up in one shot (vbmi_lookup256 per plane), the aux column's +//! four 7-bit codes per row become sign bytes through one ksigns VPERMI2B; per row group and weight octet a +//! constant two-source shuffle places each row's code bytes in its qword, VPMULTISHIFTQB spreads the symbols +//! into bytes, one vpshufb maps them to magnitudes; the signs ride the activation copy as a mask +def private emit_grid_rows_blk_vbmi(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { + let b = te.builder + let n = te.width / 64 + let colBytes = uint64(te.interleave * 4) + let w8 = te.width / 8 + var ks0 = vbmi_table_part(te, te.vb_ksigns, 0, "ks0") + var ks1 = vbmi_table_part(te, te.vb_ksigns, 1, "ks1") + var alpha = LLVMBuildLoad2Aligned(b, te.vwi8, te.vb_alpha, 64u, "alpha") + for (qd in range(te.rq)) { + let rowsOff = uint64(qd * w8) + let ncol = te.kq == 25 ? 1 : 2 + var plo : LLVMOpaqueValue? [2] + var phi : LLVMOpaqueValue? [2] + for (c in range(ncol)) { + var idx = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(2 * blk + c) * colBytes + rowsOff), ""), "gi{blk}_{c}") + plo[c] = vbmi_lookup256(te, te.vb_plane[0], idx, "plo{blk}_{c}") + phi[c] = vbmi_lookup256(te, te.vb_plane[1], idx, "phi{blk}_{c}") + } + let auxCol = te.kq == 34 ? 16 + blk : 2 * blk + 1 + var aux = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(auxCol) * colBytes + rowsOff), ""), "aux{blk}") + var codes = LLVMBuildAnd(b, vbmi_mshift(te, aux, [0, 7, 14, 21, 32, 39, 46, 53], "sc{blk}"), splat_i8w(te, 127), "") + var sgn = vbmi_permi2b(te, ks0, codes, ks1, "sg{blk}") + var acc : LLVMOpaqueValue? [2] + var acc1 : LLVMOpaqueValue? [2] + for (g in range(2)) { + acc[g] = LLVMConstNull(te.vni32) + acc1[g] = LLVMConstNull(te.vni32) + } + for (l in range(4)) { + var xv = bcast_qword_x(te, LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32 + l * 8)), ""), "x8{blk}_{l}") + for (g in range(2)) { + let r0 = g * n + var w : LLVMOpaqueValue? + if (te.kq == 25) { + let sel <- [for (k in range(w8)); (k % 8 == 1 ? w8 : 0) + (r0 + k / 8) * 4 + l] + var q = LLVMBuildShuffleVector(b, te.types, plo[0], phi[0], sel, "wq{blk}_{l}_{g}") + q = LLVMBuildAnd(b, vbmi_mshift(te, q, [0, 2, 4, 6, 8, 10, 12, 14], "ws{blk}_{l}_{g}"), splat_i8w(te, 3), "") + var sargs <- [alpha, q] + w = LLVMBuildCall2(b, te.shufb_ty, te.shufb_decl, sargs, "w{blk}_{l}_{g}") + } else { + let c = l / 2 + let p0 = (2 * l) % 4 + let sel <- [for (k in range(w8)); (k % 8 == 1 || k % 8 == 3 ? w8 : 0) + (r0 + k / 8) * 4 + p0 + (k % 8 == 2 || k % 8 == 3 ? 1 : 0)] + var q = LLVMBuildShuffleVector(b, te.types, plo[c], phi[c], sel, "wq{blk}_{l}_{g}") + q = LLVMBuildAnd(b, vbmi_mshift(te, q, [0, 3, 8, 11, 16, 19, 24, 27], "ws{blk}_{l}_{g}"), splat_i8w(te, 7), "") + var sargs <- [alpha, q] + w = LLVMBuildCall2(b, te.shufb_ty, te.shufb_decl, sargs, "w{blk}_{l}_{g}") + } + let ssel <- [for (k in range(w8)); (r0 + k / 8) * 4 + l] + var sq = LLVMBuildShuffleVector(b, te.types, sgn, sgn, ssel, "sq{blk}_{l}_{g}") + sq = LLVMBuildAnd(b, vbmi_mshift(te, sq, [0, 1, 2, 3, 4, 5, 6, 7], "sb{blk}_{l}_{g}"), splat_i8w(te, 1), "") + var m = LLVMBuildSub(b, LLVMConstNull(te.vwi8), sq, "sm{blk}_{l}_{g}") + let hi = iq2 && l >= 2 + var d = rows_dot_masked(te, hi ? acc1[g] : acc[g], w, m, xv) + if (hi) { + acc1[g] = d + } else { + acc[g] = d + } + } + } + a[qd] = LLVMBuildAdd(b, a[qd], pairwise_add_i32(te, acc[0], acc[1], "ap{blk}_{qd}"), "") + if (iq2) { + a1[qd] = LLVMBuildAdd(b, a1[qd], pairwise_add_i32(te, acc1[0], acc1[1], "ap1{blk}_{qd}"), "") + } + } +} + //! One block of a grid format's decode as ROW GROUPS: a weight-width vector is width/64 rows x 8 weights //! straight from the grid words (one u64 grid entry per iq2 row), signed by +-1 bytes - the ksigns table row //! per code or the plane's sign column (iq3s/iq2s); one pairwise add per group pair yields the fold's per-row lanes. @@ -1869,12 +2080,14 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var a1[i] = LLVMConstNull(te.vni32) } var signCol : LLVMOpaqueValue? [2] // this block's sign bytes, rows qd*rv.. at stride 4 - the gemv negates per vector, the tile's panel arrives signed - if (gather && (!pairs || te.kq == 33 || te.kq == 23 || te.dotKind != DOT_SDOT)) { + if (gather && !(pairs && te.vbmi) && (!pairs || te.kq == 33 || te.kq == 23 || te.dotKind != DOT_SDOT)) { for (qd in range(rq)) { signCol[qd] = kq_sign_bytes(te, wbP, blk, qd, "sgc{blk}_{qd}") } } - if (pairs) { + if (pairs && te.vbmi) { + emit_grid_rows_blk_vbmi(te, wbP, xb, blk, a, a1, iq2) + } elif (pairs) { te.gridRows = true emit_grid_rows_blk(te, wbP, xb, blk, signCol, a, a1, iq2) te.gridRows = false @@ -2205,6 +2418,17 @@ def private setup_tile_emit(var te : TileEmit; var gc : LlvmCodeCtx; p : TilePer var redArgTypes <- [te.vni32] te.reduce_ty = LLVMFunctionType(te.types.t_int32, redArgTypes) } + te.vbmi = p.grid == "vbmi" + if (te.vbmi) { + var b3Types <- [te.vwi8, te.vwi8, te.vwi8] + te.permi2b_decl = LLVMGetIntrinsicDeclaration(mod, LLVMLookupIntrinsicID("llvm.x86.avx512.vpermi2var.qi.512"), noTypes) + te.mshift_decl = LLVMGetIntrinsicDeclaration(mod, LLVMLookupIntrinsicID("llvm.x86.avx512.pmultishift.qb.512"), noTypes) + te.shufb_decl = LLVMGetIntrinsicDeclaration(mod, LLVMLookupIntrinsicID("llvm.x86.avx512.pshuf.b.512"), noTypes) + if (te.permi2b_decl == null || te.mshift_decl == null || te.shufb_decl == null) return false + te.permi2b_ty = LLVMFunctionType(te.vwi8, b3Types) + te.mshift_ty = LLVMFunctionType(te.vwi8, b2Types) + te.shufb_ty = LLVMFunctionType(te.vwi8, b2Types) + } if (te.dotKind == DOT_MADDUBS) { let maddId = LLVMLookupIntrinsicID(te.width == 256 ? "llvm.x86.avx2.pmadd.ub.sw" : "llvm.x86.avx512.pmaddubs.w.512") te.madd_decl = LLVMGetIntrinsicDeclaration(mod, maddId, noTypes) @@ -3099,6 +3323,9 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { iq3xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) } sign_pm1_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + if (te.vbmi) { + vbmi_emit_globals(te, LLVMGetGlobalParent(gc.impl)) + } te.iq3s_panel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "iq3s.panel") LLVMSetAlignment(te.iq3s_panel, 64u) // a cache line: at 32 half the width-512 panel loads straddled one } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index cdc72fdcd6..49420fe8dc 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -771,6 +771,7 @@ def iq3xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's tune_companion(fn = "iq3xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq3xxs_gemv"), tune_companion(fn = "iq3xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3xxs_tile", @@ -1124,6 +1125,7 @@ def iq2xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's tune_companion(fn = "iq2xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xxs_gemv"), tune_companion(fn = "iq2xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xxs_tile", diff --git a/modules/dasLLVM/daslib/llvm_jit_common.das b/modules/dasLLVM/daslib/llvm_jit_common.das index 5faf21bebc..a6088b10a6 100644 --- a/modules/dasLLVM/daslib/llvm_jit_common.das +++ b/modules/dasLLVM/daslib/llvm_jit_common.das @@ -430,6 +430,7 @@ var public g_target_x64_f16c = false var public g_target_x64_vnni256 = false var public g_target_x64_avx512bw = false var public g_target_x64_avx512vnni = false +var public g_target_x64_avx512vbmi = false // zmm byte permutes across two registers (VPERMI2B) + VPMULTISHIFTQB: the grid formats' symbol lattice // AVX-VNNI-INT8 (VEX vpdpbssd — native s8×s8, no sign trick). No shipped silicon in the fleet // yet; lights up via cpuid on future boxes and via DAS_JIT_X64_FORCE_FEATURES for emission-only // verification today. @@ -626,6 +627,7 @@ def public init_jit_target_flags(target_triple : string = ""; host_features : bo || (x64_tier_feature("avx512vnni", forced_only) && x64_tier_feature("avx512vl", forced_only))) g_target_x64_avx512bw = g_target_x64_avx2 && x64_tier_feature("avx512f", forced_only) && x64_tier_feature("avx512bw", forced_only) g_target_x64_avx512vnni = g_target_x64_avx512bw && x64_tier_feature("avx512vnni", forced_only) + g_target_x64_avx512vbmi = g_target_x64_avx512bw && x64_tier_feature("avx512vbmi", forced_only) g_target_x64_vnniint8 = g_target_x64_avx2 && x64_tier_feature("avxvnniint8", forced_only) g_target_x64_amx = g_target_is_x64 && x64_tier_feature("amx-tile", forced_only) && x64_tier_feature("amx-int8", forced_only) // target OS truth (native = host platform; cross = the triple's OS field) diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index b4fffc9edd..46cc1cc53f 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -36,7 +36,7 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x66ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x67ul // the grid formats' gemv applies signs as one masked negate per weight vector (0x59: aarch64 hosts append +i8mm when cpu_supports says so) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index d55a51c8a9..6188de5852 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -1237,7 +1237,7 @@ def tune_pick_fallback(chain : string) : string { return "" } -let private TUNE_KNOWN_FEATURES = "avx2;f16c;fma;sse4.2;avxvnni;avxvnniint8;avx512f;avx512bw;avx512vl;avx512vnni;amx-tile;amx-int8;neon;dotprod;i8mm;bf16;fullfp16;sve;lse" +let private TUNE_KNOWN_FEATURES = "avx2;f16c;fma;sse4.2;avxvnni;avxvnniint8;avx512f;avx512bw;avx512vl;avx512vnni;avx512vbmi;amx-tile;amx-int8;neon;dotprod;i8mm;bf16;fullfp16;sve;lse" //! The ';'-joined subset of the known `requires=` feature names this box passes — stamped //! into every sidecar's provenance as `features`, which is what lets a shipped profile say diff --git a/src/builtin/module_builtin_runtime.cpp b/src/builtin/module_builtin_runtime.cpp index b52a590b8e..c7a5f83cb4 100644 --- a/src/builtin/module_builtin_runtime.cpp +++ b/src/builtin/module_builtin_runtime.cpp @@ -2173,6 +2173,7 @@ namespace das if ( strcmp(feature, "avx512bw")==0 ) return os_zmm && (r7[1] & (1<<30)) != 0; if ( strcmp(feature, "avx512vl")==0 ) return os_zmm && (r7[1] & (1u<<31)) != 0; if ( strcmp(feature, "avx512vnni")==0 ) return os_zmm && (r7[2] & (1<<11)) != 0; + if ( strcmp(feature, "avx512vbmi")==0 ) return os_zmm && (r7[2] & (1<<1)) != 0; // AMX names use the LLVM hyphen spelling so cpuid names == target-feature names // (DAS_JIT_X64_FORCE_FEATURES / llc -mattr pass them through verbatim). XCR0 tile // bits are kernel-boot truth; the per-process XTILEDATA grant (Linux arch_prctl) From e52fd318a02ab5de38abc363cfd46521d72367d1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:32:10 -0700 Subject: [PATCH 077/123] vbmi lattice: the iq3 planes pack two weights each (2 x 3 bits), not four - the 12-bit entry split across the planes --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 2c19ed453f..b63cda2a32 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1798,16 +1798,16 @@ def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { p0 |> reserve(256) p1 |> reserve(256) if (te.kq == 25) { - let g = iq2xxs_grid2() + let g = iq2xxs_grid2() // an entry is two words: weights 0-3, 4-7 - one plane each, 4 x 2 bits for (e in range(256)) { - p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e], alpha, bits)), 0)) - p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e + 1], alpha, bits)), 0)) + p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e], alpha, bits, 4)), 0)) + p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e + 1], alpha, bits, 4)), 0)) } } else { - let g = iq3xxs_grid() + let g = iq3xxs_grid() // an entry is one word of 4 weights: weights 0-1 and 2-3 per plane, 2 x 3 bits for (e in range(256)) { - p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e], alpha, bits) & 255), 0)) - p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e] >> 16u, alpha, bits) & 255), 0)) + p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e], alpha, bits, 2)), 0)) + p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e] >> 16u, alpha, bits, 2)), 0)) } } te.vb_plane[0] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p0", p0) @@ -1827,10 +1827,10 @@ def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { te.vb_ksigns = vbmi_const_bytes(te, mod, "dasllama.vbmi.ksigns", kv) } -//! a grid word's four weight bytes as packed symbols (weight t at bits t*bits); a byte outside the alphabet is a broken table -def private vbmi_pack_word(word : uint; alpha : array; bits : int) : int { +//! a grid word's low `nbytes` weight bytes as packed symbols (weight t at bits t*bits); a byte outside the alphabet is a broken table +def private vbmi_pack_word(word : uint; alpha : array; bits, nbytes : int) : int { var code = 0 - for (t in range(4)) { + for (t in range(nbytes)) { let wb = int((word >> uint(8 * t)) & 0xFFu) let si = alpha |> find_index(wb) if (si < 0) { From 2a6af045c031bd10970259efa8d390bbe16433a5 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:37:19 -0700 Subject: [PATCH 078/123] vbmi lattice: iq3s, iq2xs and iq2s join (512 / 512 / 1024-entry grids); the lookup blends the 256-entry halves on the index's 9th and 10th bits vbmi_lookup runs VPERMI2B per 128 entries, blends pairs on index bit 7, then on the caller's lane masks - iq3s and iq2s take theirs from the row's qh byte broadcast over its lanes, iq2xs from bit 0 of the u16 word's high byte. Signs: iq3s / iq2s read the plane's own sign column (already in the lane layout), iq2xs its 7-bit codes (the high byte >> 1) through the ksigns VPERMI2B. Compiles and declines on zen2; the five families' TEST and races run on the VBMI boxes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 196 +++++++++++++----- .../dasLLAMA/dasllama/dasllama_math_gen.das | 3 + 2 files changed, 144 insertions(+), 55 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index b63cda2a32..953b8a9f0d 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1777,38 +1777,56 @@ def private row_weights8(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l } //! the lattice's constant tables, emitted once per module: the grid's two compact code planes (entry e's -//! low / high symbols, 2 bits each for the iq2 alphabet, 3 for iq3), the alphabet as a per-lane vpshufb -//! table, and ksigns_iq2xs whole (128 bytes: exactly the two registers one VPERMI2B indexes) +//! low / high half - four 2-bit symbols for the iq2 alphabet, two 3-bit symbols for iq3), the alphabet as a +//! per-lane vpshufb table, and ksigns_iq2xs whole (128 bytes: exactly the two registers one VPERMI2B indexes) def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { - let tag = te.kq == 25 ? "iq2xxs" : "iq3xxs" + let tag = vbmi_tag(te.kq) te.vb_plane[0] = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.p0") te.vb_plane[1] = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.p1") te.vb_alpha = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.alpha") te.vb_ksigns = LLVMGetNamedGlobal(mod, "dasllama.vbmi.ksigns") if (te.vb_plane[0] != null && te.vb_plane[1] != null && te.vb_alpha != null && te.vb_ksigns != null) return + let iq3 = te.kq == 33 || te.kq == 34 var alpha : array - if (te.kq == 25) { - alpha <- [8, 25, 43] - } else { + if (te.kq == 34) { alpha <- [2, 6, 10, 14, 18, 22, 26, 31] + } elif (te.kq == 33) { + alpha <- [1, 3, 5, 7, 9, 11, 13, 15] + } else { + alpha <- [8, 25, 43] } - let bits = te.kq == 25 ? 2 : 3 + let bits = iq3 ? 3 : 2 + let entries = vbmi_entries(te.kq) var p0 : array var p1 : array - p0 |> reserve(256) - p1 |> reserve(256) - if (te.kq == 25) { - let g = iq2xxs_grid2() // an entry is two words: weights 0-3, 4-7 - one plane each, 4 x 2 bits - for (e in range(256)) { - p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e], alpha, bits, 4)), 0)) - p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e + 1], alpha, bits, 4)), 0)) + p0 |> reserve(entries) + p1 |> reserve(entries) + if (iq3) { // an entry is one word of 4 weights: weights 0-1 and 2-3 per plane + var g : array + if (te.kq == 34) { + g <- [for (w in iq3xxs_grid()); w] + } else { + g <- [for (w in iq3s_grid()); w] } - } else { - let g = iq3xxs_grid() // an entry is one word of 4 weights: weights 0-1 and 2-3 per plane, 2 x 3 bits - for (e in range(256)) { + for (e in range(entries)) { p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e], alpha, bits, 2)), 0)) p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e] >> 16u, alpha, bits, 2)), 0)) } + delete g + } else { // an entry is two words: weights 0-3, 4-7 - one plane each + var g : array + if (te.kq == 25) { + g <- [for (w in iq2xxs_grid2()); w] + } elif (te.kq == 24) { + g <- [for (w in iq2xs_grid2()); w] + } else { + g <- [for (w in iq2s_grid2()); w] + } + for (e in range(entries)) { + p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e], alpha, bits, 4)), 0)) + p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e + 1], alpha, bits, 4)), 0)) + } + delete g } te.vb_plane[0] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p0", p0) te.vb_plane[1] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p1", p1) @@ -1818,15 +1836,22 @@ def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { al |> push(LLVMConstInt(te.types.t_int8, uint64(k % 16 < length(alpha) ? alpha[k % 16] : 0), 0)) } te.vb_alpha = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.alpha", al) - let ks = ksigns_iq2xs() - var kv : array - kv |> reserve(128) - for (k in range(128)) { - kv |> push(LLVMConstInt(te.types.t_int8, uint64(ks[k]), 0)) + if (te.vb_ksigns == null) { + let ks = ksigns_iq2xs() + var kv : array + kv |> reserve(128) + for (k in range(128)) { + kv |> push(LLVMConstInt(te.types.t_int8, uint64(ks[k]), 0)) + } + te.vb_ksigns = vbmi_const_bytes(te, mod, "dasllama.vbmi.ksigns", kv) } - te.vb_ksigns = vbmi_const_bytes(te, mod, "dasllama.vbmi.ksigns", kv) } +def private vbmi_tag(kq : int) : string => kq == 25 ? "iq2xxs" : (kq == 24 ? "iq2xs" : (kq == 23 ? "iq2s" : (kq == 34 ? "iq3xxs" : "iq3s"))) + +//! grid entries per format: the compact plane's byte count +def private vbmi_entries(kq : int) : int => (kq == 25 || kq == 34) ? 256 : (kq == 23 ? 1024 : 512) + //! a grid word's low `nbytes` weight bytes as packed symbols (weight t at bits t*bits); a byte outside the alphabet is a broken table def private vbmi_pack_word(word : uint; alpha : array; bits, nbytes : int) : int { var code = 0 @@ -1873,13 +1898,25 @@ def private vbmi_mshift(var te : TileEmit; var v : LLVMOpaqueValue?; ctrl : arra return LLVMBuildCall2(te.builder, te.mshift_ty, te.mshift_decl, args, name) } -//! 64 byte indices through a 256-entry byte table: two VPERMI2B (128 entries each) and one blend on index bit 7 -def private vbmi_lookup256(var te : TileEmit; var tbl, idx : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { +//! 64 indices through a byte table of `entries` (256 / 512 / 1024): VPERMI2B covers 128 entries per pair of +//! table registers, index bit 7 blends the pairs, and the caller's masks (the index's 9th and 10th bits, per +//! lane) blend the 256-entry halves above that +def private vbmi_lookup(var te : TileEmit; var tbl : LLVMOpaqueValue?; entries : int; var idx : LLVMOpaqueValue?; var masks : array; name : string) : LLVMOpaqueValue? { let b = te.builder - var t0 = vbmi_permi2b(te, vbmi_table_part(te, tbl, 0, "{name}.t0"), idx, vbmi_table_part(te, tbl, 1, "{name}.t1"), "{name}.lo") - var t1 = vbmi_permi2b(te, vbmi_table_part(te, tbl, 2, "{name}.t2"), idx, vbmi_table_part(te, tbl, 3, "{name}.t3"), "{name}.hi") + var level : array + for (k in range(entries / 128)) { + level |> push(vbmi_permi2b(te, vbmi_table_part(te, tbl, 2 * k, "{name}.t{2 * k}"), idx, vbmi_table_part(te, tbl, 2 * k + 1, "{name}.t{2 * k + 1}"), "{name}.l{k}")) + } var high = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, idx, LLVMConstNull(te.vwi8), "{name}.b7") - return LLVMBuildSelect(b, high, t1, t0, name) + var depth = 0 + while (length(level) > 1) { + var sel = depth == 0 ? high : masks[depth - 1] + var nxt <- [for (j in range(length(level) / 2)); LLVMBuildSelect(b, sel, level[2 * j + 1], level[2 * j], "{name}.s{depth}_{j}")] + level |> clear() + level <- nxt + depth++ + } + return level[0] } //! the unsigned-weight dot with the signs as a byte mask on the activation copy: x' = (x ^ m) - m @@ -1896,33 +1933,84 @@ def private rows_dot_masked(var te : TileEmit; var acc, w, m, xv : LLVMOpaqueVal return LLVMBuildAdd(b, acc, LLVMBuildCall2(b, te.maddwd_ty, te.maddwd_decl, wargs, "quads"), "dot") } -//! One block of iq2xxs / iq3xxs as ROW GROUPS through the VBMI symbol lattice: the block's index column -//! (16 rows x 4 bytes) looks its compact codes up in one shot (vbmi_lookup256 per plane), the aux column's -//! four 7-bit codes per row become sign bytes through one ksigns VPERMI2B; per row group and weight octet a -//! constant two-source shuffle places each row's code bytes in its qword, VPMULTISHIFTQB spreads the symbols -//! into bytes, one vpshufb maps them to magnitudes; the signs ride the activation copy as a mask -def private emit_grid_rows_blk_vbmi(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { +//! the block's 64-byte column `col` for rows qd*rv.. (row r's four bytes at r*4) +def private vbmi_col(var te : TileEmit; var wbP : LLVMOpaqueValue?; col, qd : int; name : string) : LLVMOpaqueValue? { + let off = uint64(col * te.interleave * 4 + qd * (te.width / 8)) + return load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(te.builder, wbP, te.types->ConstI64(off), ""), name) +} + +//! the row's qh byte for this block (packed byte position 64 + blk: column 16 + blk/4, byte blk%4) broadcast +//! over its four lanes, then bit `bitOf(lane%4)` as a lane mask - the iq3s / iq2s index bits above the qs byte +def private vbmi_qh_mask(var te : TileEmit; var qhCol : LLVMOpaqueValue?; blk : int; bitOf : block<(p : int) : int>; name : string) : LLVMOpaqueValue? { let b = te.builder - let n = te.width / 64 - let colBytes = uint64(te.interleave * 4) let w8 = te.width / 8 + let idx <- [for (k in range(w8)); (k / 4) * 4 + blk % 4] + var bytes = LLVMBuildShuffleVector(b, te.types, qhCol, qhCol, idx, "{name}.q") + var bits <- [for (k in range(w8)); LLVMConstInt(te.types.t_int8, uint64(1 << invoke(bitOf, k % 4)), 0)] + var bitsv = LLVMConstVector(array_data_ptr(bits), uint(w8)) + return LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, LLVMBuildAnd(b, bytes, bitsv, ""), LLVMConstNull(te.vwi8), name) +} + +//! the row's sign bytes from its four 7-bit ksigns codes (lane r*4+l), parity included: one VPERMI2B over the table +def private vbmi_ksigns(var te : TileEmit; var codes : LLVMOpaqueValue?; name : string) : LLVMOpaqueValue? { var ks0 = vbmi_table_part(te, te.vb_ksigns, 0, "ks0") var ks1 = vbmi_table_part(te, te.vb_ksigns, 1, "ks1") + return vbmi_permi2b(te, ks0, LLVMBuildAnd(te.builder, codes, splat_i8w(te, 127), ""), ks1, name) +} + +//! One block of a grid format as ROW GROUPS through the VBMI symbol lattice. Per format the block's index +//! bytes are gathered into one 64-lane vector per column (lane r*4 + position), looked up in the compact code +//! planes (vbmi_lookup; the 9th / 10th index bits arrive as lane masks), and the row's sign bytes land in the +//! same lane layout (the plane's own column for iq3s / iq2s, one ksigns VPERMI2B over the 7-bit codes for the +//! rest). Per row group and weight octet a constant two-source shuffle places each row's code bytes in its +//! qword, VPMULTISHIFTQB spreads the symbols into bytes, one vpshufb maps them to magnitudes; the signs ride the +//! activation copy as a mask. +def private emit_grid_rows_blk_vbmi(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { // nolint:STYLE037,STYLE038 — one block's decode: the five formats' index assembly and the shared expansion + let b = te.builder + let n = te.width / 64 + let w8 = te.width / 8 + let iq3 = te.kq == 33 || te.kq == 34 + let entries = vbmi_entries(te.kq) var alpha = LLVMBuildLoad2Aligned(b, te.vwi8, te.vb_alpha, 64u, "alpha") for (qd in range(te.rq)) { - let rowsOff = uint64(qd * w8) - let ncol = te.kq == 25 ? 1 : 2 + let ncol = iq3 ? 2 : 1 var plo : LLVMOpaqueValue? [2] var phi : LLVMOpaqueValue? [2] + var sgn : LLVMOpaqueValue? for (c in range(ncol)) { - var idx = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(2 * blk + c) * colBytes + rowsOff), ""), "gi{blk}_{c}") - plo[c] = vbmi_lookup256(te, te.vb_plane[0], idx, "plo{blk}_{c}") - phi[c] = vbmi_lookup256(te, te.vb_plane[1], idx, "phi{blk}_{c}") - } - let auxCol = te.kq == 34 ? 16 + blk : 2 * blk + 1 - var aux = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wbP, te.types->ConstI64(uint64(auxCol) * colBytes + rowsOff), ""), "aux{blk}") - var codes = LLVMBuildAnd(b, vbmi_mshift(te, aux, [0, 7, 14, 21, 32, 39, 46, 53], "sc{blk}"), splat_i8w(te, 127), "") - var sgn = vbmi_permi2b(te, ks0, codes, ks1, "sg{blk}") + var idx : LLVMOpaqueValue? + var masks : array + if (te.kq == 25 || te.kq == 34) { + idx = vbmi_col(te, wbP, 2 * blk + c, qd, "gi{blk}_{c}") + } elif (te.kq == 33) { + idx = vbmi_col(te, wbP, 2 * blk + c, qd, "gi{blk}_{c}") + var qh = vbmi_col(te, wbP, 16 + blk / 4, qd, "qh{blk}") + masks |> push(vbmi_qh_mask(te, qh, blk, $(p : int) : int => 4 * c + p, "qm{blk}_{c}")) + } elif (te.kq == 23) { + idx = vbmi_col(te, wbP, blk, qd, "gi{blk}") + var qh = vbmi_col(te, wbP, 16 + blk / 4, qd, "qh{blk}") + masks |> push(vbmi_qh_mask(te, qh, blk, $(p : int) : int => 2 * p, "qm{blk}_0")) + masks |> push(vbmi_qh_mask(te, qh, blk, $(p : int) : int => 2 * p + 1, "qm{blk}_1")) + sgn = vbmi_col(te, wbP, 8 + blk, qd, "sg{blk}") + } else { // 24: u16 words - columns 2blk (l 0, 1) and 2blk+1 (l 2, 3), lane r*4+l takes word l's low then high byte + var ca = vbmi_col(te, wbP, 2 * blk, qd, "wa{blk}") + var cb = vbmi_col(te, wbP, 2 * blk + 1, qd, "wb{blk}") + let lo <- [for (k in range(w8)); (k % 4 < 2 ? 0 : w8) + (k / 4) * 4 + 2 * (k % 2)] + let hi <- [for (k in range(w8)); (k % 4 < 2 ? 0 : w8) + (k / 4) * 4 + 2 * (k % 2) + 1] + idx = LLVMBuildShuffleVector(b, te.types, ca, cb, lo, "gi{blk}") + var hib = LLVMBuildShuffleVector(b, te.types, ca, cb, hi, "gh{blk}") + masks |> push(LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, LLVMBuildAnd(b, hib, splat_i8w(te, 1), ""), LLVMConstNull(te.vwi8), "qm{blk}")) + sgn = vbmi_ksigns(te, LLVMBuildLShr(b, hib, splat_i8w(te, 1), ""), "sg{blk}") + } + plo[c] = vbmi_lookup(te, te.vb_plane[0], entries, idx, masks, "plo{blk}_{c}") + phi[c] = vbmi_lookup(te, te.vb_plane[1], entries, idx, masks, "phi{blk}_{c}") + } + if (te.kq == 33) { + sgn = vbmi_col(te, wbP, 18 + blk, qd, "sg{blk}") + } elif (te.kq == 25 || te.kq == 34) { // the aux32's four 7-bit codes per row, two rows per qword + var aux = vbmi_col(te, wbP, te.kq == 34 ? 16 + blk : 2 * blk + 1, qd, "aux{blk}") + sgn = vbmi_ksigns(te, vbmi_mshift(te, aux, [0, 7, 14, 21, 32, 39, 46, 53], "sc{blk}"), "sg{blk}") + } var acc : LLVMOpaqueValue? [2] var acc1 : LLVMOpaqueValue? [2] for (g in range(2)) { @@ -1933,22 +2021,20 @@ def private emit_grid_rows_blk_vbmi(var te : TileEmit; var wbP, xb : LLVMOpaqueV var xv = bcast_qword_x(te, LLVMBuildAdd(b, xb, te.types->ConstI64(uint64(blk * 32 + l * 8)), ""), "x8{blk}_{l}") for (g in range(2)) { let r0 = g * n - var w : LLVMOpaqueValue? - if (te.kq == 25) { + var q : LLVMOpaqueValue? + if (!iq3) { // qword i <- row r0+i's low-plane byte, high-plane byte (the rest don't care) let sel <- [for (k in range(w8)); (k % 8 == 1 ? w8 : 0) + (r0 + k / 8) * 4 + l] - var q = LLVMBuildShuffleVector(b, te.types, plo[0], phi[0], sel, "wq{blk}_{l}_{g}") + q = LLVMBuildShuffleVector(b, te.types, plo[0], phi[0], sel, "wq{blk}_{l}_{g}") q = LLVMBuildAnd(b, vbmi_mshift(te, q, [0, 2, 4, 6, 8, 10, 12, 14], "ws{blk}_{l}_{g}"), splat_i8w(te, 3), "") - var sargs <- [alpha, q] - w = LLVMBuildCall2(b, te.shufb_ty, te.shufb_decl, sargs, "w{blk}_{l}_{g}") - } else { + } else { // qword i <- the two entries (h = 0, 1) of row r0+i, low then high plane each let c = l / 2 let p0 = (2 * l) % 4 let sel <- [for (k in range(w8)); (k % 8 == 1 || k % 8 == 3 ? w8 : 0) + (r0 + k / 8) * 4 + p0 + (k % 8 == 2 || k % 8 == 3 ? 1 : 0)] - var q = LLVMBuildShuffleVector(b, te.types, plo[c], phi[c], sel, "wq{blk}_{l}_{g}") + q = LLVMBuildShuffleVector(b, te.types, plo[c], phi[c], sel, "wq{blk}_{l}_{g}") q = LLVMBuildAnd(b, vbmi_mshift(te, q, [0, 3, 8, 11, 16, 19, 24, 27], "ws{blk}_{l}_{g}"), splat_i8w(te, 7), "") - var sargs <- [alpha, q] - w = LLVMBuildCall2(b, te.shufb_ty, te.shufb_decl, sargs, "w{blk}_{l}_{g}") } + var sargs <- [alpha, q] + var w = LLVMBuildCall2(b, te.shufb_ty, te.shufb_decl, sargs, "w{blk}_{l}_{g}") let ssel <- [for (k in range(w8)); (r0 + k / 8) * 4 + l] var sq = LLVMBuildShuffleVector(b, te.types, sgn, sgn, ssel, "sq{blk}_{l}_{g}") sq = LLVMBuildAnd(b, vbmi_mshift(te, sq, [0, 1, 2, 3, 4, 5, 6, 7], "sb{blk}_{l}_{g}"), splat_i8w(te, 1), "") diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 49420fe8dc..add6bb0b7a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -695,6 +695,7 @@ def iq3sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's tune_companion(fn = "iq3sq8_gemv_gen", gen = "dasllama_gemm_gen::iq3s_gemv"), tune_companion(fn = "iq3sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3s_tile", @@ -979,6 +980,7 @@ def iq2sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's tune_companion(fn = "iq2sq8_gemv_gen", gen = "dasllama_gemm_gen::iq2s_gemv"), tune_companion(fn = "iq2sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2s_tile", @@ -1051,6 +1053,7 @@ def iq2xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's tune_companion(fn = "iq2xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xs_gemv"), tune_companion(fn = "iq2xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xs_tile", From 9ec45a65cc137181785f36ffd0702d50ff343c50 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 11:40:14 -0700 Subject: [PATCH 079/123] kq_kernel_bench: --chunks-per-lane sets the engine splitter's gemv grain for the team arm --- modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 818c15be74..512b79e907 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -55,6 +55,8 @@ struct BenchArgs { team : bool @clarg_doc = "Append every round's sample (us, in round order) as one more column - the per-round view of a bimodal row" each : bool + @clarg_doc = "--team: gemv chunks per lane for the engine's splitter (0 = the engine default, 8; 1 = one equal chunk per lane)" + chunks_per_lane : int = 0 } typedef GemvFn = function<(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void> @@ -638,6 +640,9 @@ def main : int { BASE_ALIGN = int64(cfg.base_align) BASE_OFFSET = int64(cfg.base_offset) g_each_round = cfg.each + if (cfg.chunks_per_lane > 0) { + set_gemv_chunks_per_lane(cfg.chunks_per_lane) + } return rc if (rc >= 0) if (!jit_enabled()) { to_log(LOG_ERROR, "run with -jit: the kq generators only fire under the JIT\n") From 03864eaec7193d18d574ef17e92fc3781a959923 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 12:05:03 -0700 Subject: [PATCH 080/123] kernel parity plan: the mr16 race verdict on both boxes; the Intel k6 warm-up find --- plans/kernel_parity_pass.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 660e21d2ae..d7299620b1 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -830,5 +830,14 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. Same run, a find: on zen4 the row form beats the panel form for iq2xs at 16 lanes (2194 vs the ladder's panel 2559 us - 0.91x of the reference instead of 0.78x) where the class gate picks the panel; the forms probe (zen4_forms.sh, both forms x five grids x two shapes) decides the gate. +- 2026-09-01: the width256_mr16 row raced alone on both boxes (post probes, d=32768, tune mode; full tables + tmp zen4_post.log / intel_post.log). At 16 lanes every seat of every format sits within ~5% on both boxes - + the minted crowns stand at the engine shape. One-thread streamed decode: on Intel 256_mr16 beats the 512 + gemv for k3 (12214 vs 16462), k5 (18972 vs 21184) and k6 (27321 vs 31723) while its tile loses ~14%, so + the gemv's own seat is expected to take it at the next Intel mint; on zen4 it loses everywhere (k3 16193 + vs 7280). Intel iq2s: ALL seats ~0.90-0.94 of the reference at 16 lanes - the seat is not iq2s's lever. +- 2026-09-01: Intel k6 one thread, 40 rounds --each, normal mode: 3425, 1989, then steady 1637-1642 us - + the slow reads were WARM-UP, steady state is 1.16x of the reference (1882); the ladder's 5-round + best 1831 / median 2925 sampled the ramp. The 16-lane arm decides whether the 0.83 row is the same artifact. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From cb3238e6d9be249d2a2159fa432f9907bac0cabd Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 12:06:27 -0700 Subject: [PATCH 081/123] kq_kernel_bench: three unmeasured warm rounds in the SOLO arm - the Intel k6 ramp outlasts one (3425, 1989, then steady 1637 us) --- modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 512b79e907..a3f0cee8fc 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -370,7 +370,9 @@ def perm_wanted(spec, suffix : string) : bool { def time_rows(var stats : array; rounds : int; blk : block<(row : int) : void>) { for (row in range(length(stats))) { stats[row].us |> reserve(rounds) - invoke(blk, row) // warmup + codegen, unmeasured + for (_w in range(3)) { // codegen + warm-up, unmeasured: Intel k6 ramps for ~3 rounds (3425, 1989, then steady 1637 us) - one warm call is not enough + invoke(blk, row) + } } for (_r in range(rounds)) { for (row in range(length(stats))) { From a8748f2004642017208113a8842dd3479b1c2ca3 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 12:14:11 -0700 Subject: [PATCH 082/123] kernel parity plan: the lattice's Intel verdict; the k6 gap is mode-specific --- plans/kernel_parity_pass.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index d7299620b1..9e40c38ca4 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -839,5 +839,14 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. - 2026-09-01: Intel k6 one thread, 40 rounds --each, normal mode: 3425, 1989, then steady 1637-1642 us - the slow reads were WARM-UP, steady state is 1.16x of the reference (1882); the ladder's 5-round best 1831 / median 2925 sampled the ramp. The 16-lane arm decides whether the 0.83 row is the same artifact. +- 2026-09-01: the VBMI lattice is CORRECT and WINS (Intel c8i, first races): TEST bit-exact on all five + grid families (grid_vbmi maxdiff <= 2e-6); iq2xxs one thread 19334 vs the 512 row's 21823 us, 16 lanes + 2249 vs 2599 (the reference 4254 - 1.89x); iq3xxs one thread 30438 vs ~39700 (the panel form's post-probe + read). The vbmi row's tile is the 512 body, so the tile race ties and the gemv's own seat crowns the + lattice where it wins - a plain re-mint adopts it, no framework change. +- 2026-09-01: Intel k6 16-lane mode gap CONFIRMED mode-specific, not warm-up: same pinned seat, steady state + ~4900-5300 us normal vs ~3236-3400 tune over 20 rounds; one thread differs 1% (1629 vs 1616). Arena phase + is an 8% one-thread effect (off 64 best 1764, off 128-256 ~1920), not the 1.5x. intel_k6perf.sh counts the + steady loop (perf stat + record by DSO) in both modes - kernel pages vs dispatcher decides the next step. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 7544ae37d3df8fea47590d335772521b280d45a2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 12:24:03 -0700 Subject: [PATCH 083/123] kernel parity plan: the lattice's full Intel race - every grid row clears 0.95 --- plans/kernel_parity_pass.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 9e40c38ca4..867ba88791 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -848,5 +848,10 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. ~4900-5300 us normal vs ~3236-3400 tune over 20 rounds; one thread differs 1% (1629 vs 1616). Arena phase is an 8% one-thread effect (off 64 best 1764, off 128-256 ~1920), not the 1.5x. intel_k6perf.sh counts the steady loop (perf stat + record by DSO) in both modes - kernel pages vs dispatcher decides the next step. +- 2026-09-01: the lattice's full Intel race (tune mode, d=32768; vbmi / old 512 seat / the reference at + 16 threads): 16 lanes iq2xxs 2249 / 2599 / 4255, iq3xxs 3376 / 4536 / 6704, iq3s 3726 / 7212 / 7413, + iq2xs 2815 / 3951 / 4610, iq2s 3316 / 5706 / 3848; one thread iq2xxs 19334 / 21823, iq3xxs 30438 / 39773, + iq3s 36485 / 72793, iq2xs 23533 / 32658, iq2s 28428 / 36371 us. Every Intel grid row clears 0.95 with the + lattice - iq2s (the 0.90 row) reads 1.16x. The zen4 half runs next; then the re-mints adopt the seats. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 853529339acb83f4fb7d761aed8717a49612f694 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 12:38:13 -0700 Subject: [PATCH 084/123] kq_kernel_bench: a 'timed rounds begin' log line before each timed loop - external profilers sync on it instead of guessing the JIT startup length --- modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index a3f0cee8fc..01d8eb84ec 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -303,6 +303,7 @@ def time_rows_team(var stats : array; rounds : int; blk : block<(row : invoke(blk, row, laneP) } } + to_log(LOG_INFO, "kq_kernel_bench: timed rounds begin\n") // the profiler sync line, as in time_rows for (_r in range(rounds)) { for (row in range(length(stats))) { for (u in laneUs) { @@ -374,6 +375,7 @@ def time_rows(var stats : array; rounds : int; blk : block<(row : int) invoke(blk, row) } } + to_log(LOG_INFO, "kq_kernel_bench: timed rounds begin\n") // external profilers key on this line to skip the JIT startup for (_r in range(rounds)) { for (row in range(length(stats))) { let t0 = ref_time_ticks() From e206ae459977e9f01a1f850affaaed3f7fa01d41 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 12:50:30 -0700 Subject: [PATCH 085/123] kernel parity plan: the lattice sweeps zen4; the Intel gap is memory-system; the cold-start crash --- plans/kernel_parity_pass.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 867ba88791..66734a7e2d 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -853,5 +853,18 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. iq2xs 2815 / 3951 / 4610, iq2s 3316 / 5706 / 3848; one thread iq2xxs 19334 / 21823, iq3xxs 30438 / 39773, iq3s 36485 / 72793, iq2xs 23533 / 32658, iq2s 28428 / 36371 us. Every Intel grid row clears 0.95 with the lattice - iq2s (the 0.90 row) reads 1.16x. The zen4 half runs next; then the re-mints adopt the seats. +- 2026-09-01: the lattice's zen4 race (tune mode, d=32768; vbmi / old 512 seat / the reference): one thread + iq2xxs 17571 / 22402 / 32946, iq3xxs 27384 / 48023 / 43551, iq3s 26120 / 61810 / 49870, iq2xs 20518 / + 36590 / 30978, iq2s 26909 / 39628 / 32855; 16 lanes best iq2xxs 1551 / 1666 / 2106, iq3xxs 2520 / 3108 / + 2810, iq3s 2778 / 3882 / 3231, iq2xs 1618 / 2334 / 2006, iq2s 1935 / 2516 / 2113 us. TEST bit-exact. + Every zen4 grid row clears 0.95 with the lattice - the four 0.76-0.90 one-thread rows read 1.2-1.9x. +- 2026-09-01: the Intel mode gap narrowed to the memory system: warm cache, both modes 96%+ of cycles in + the kernel DLL, same clock, same THP rollup, and the instruction rate ratio equals the wall ratio exactly + (identical instructions per gemv) - the normal body just stalls 1.54x more, 16 lanes only. Tune streams + 385 MB at 127 GB/s (partial cache residency), normal at 82.8. Round 5 sweeps d (8k/32k/64k), adds q8, and + reads the arena VMA's own AnonHugePages plus LLC counters per mode. +- 2026-09-01: cold-start SIGSEGV at 0xc0 right after "Library linked - ok" (normal mode, 16-lane team k6 + d=32768, Intel; reproduced 3x cold, never warm, earlier bootstrap ladders passed) - a post-link rebind + bug to chase before the PR. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 3bf5fefc0737bd11e147948b63069be91fbbce88 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 13:17:17 -0700 Subject: [PATCH 086/123] vbmi lattice: compile-time hygiene - grid tables held as fixed-array locals (no by-value temporaries in comprehensions), the lookup ladder copies instead of moving into a cleared array, the qh mask takes bit0/stride instead of a block --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 73 +++++++++++-------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 953b8a9f0d..1c57107fb5 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1795,38 +1795,36 @@ def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { } else { alpha <- [8, 25, 43] } - let bits = iq3 ? 3 : 2 let entries = vbmi_entries(te.kq) var p0 : array var p1 : array p0 |> reserve(entries) p1 |> reserve(entries) - if (iq3) { // an entry is one word of 4 weights: weights 0-1 and 2-3 per plane - var g : array - if (te.kq == 34) { - g <- [for (w in iq3xxs_grid()); w] - } else { - g <- [for (w in iq3s_grid()); w] + if (te.kq == 34) { // an iq3 entry is one word of 4 weights: weights 0-1 and 2-3 per plane + let g = iq3xxs_grid() + for (e in range(entries)) { + vbmi_push_iq3(te, p0, p1, g[e], alpha) } + } elif (te.kq == 33) { + let g = iq3s_grid() for (e in range(entries)) { - p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e], alpha, bits, 2)), 0)) - p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[e] >> 16u, alpha, bits, 2)), 0)) - } - delete g - } else { // an entry is two words: weights 0-3, 4-7 - one plane each - var g : array - if (te.kq == 25) { - g <- [for (w in iq2xxs_grid2()); w] - } elif (te.kq == 24) { - g <- [for (w in iq2xs_grid2()); w] - } else { - g <- [for (w in iq2s_grid2()); w] + vbmi_push_iq3(te, p0, p1, g[e], alpha) } + } elif (te.kq == 25) { // an iq2 entry is two words: weights 0-3, 4-7 - one plane each + let g = iq2xxs_grid2() + for (e in range(entries)) { + vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) + } + } elif (te.kq == 24) { + let g = iq2xs_grid2() for (e in range(entries)) { - p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e], alpha, bits, 4)), 0)) - p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(g[2 * e + 1], alpha, bits, 4)), 0)) + vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) + } + } else { + let g = iq2s_grid2() + for (e in range(entries)) { + vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) } - delete g } te.vb_plane[0] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p0", p0) te.vb_plane[1] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p1", p1) @@ -1847,6 +1845,16 @@ def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { } } +def private vbmi_push_iq3(var te : TileEmit; var p0, p1 : array; word : uint; alpha : array) { + p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(word, alpha, 3, 2)), 0)) + p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(word >> 16u, alpha, 3, 2)), 0)) +} + +def private vbmi_push_iq2(var te : TileEmit; var p0, p1 : array; w0, w1 : uint; alpha : array) { + p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(w0, alpha, 2, 4)), 0)) + p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(w1, alpha, 2, 4)), 0)) +} + def private vbmi_tag(kq : int) : string => kq == 25 ? "iq2xxs" : (kq == 24 ? "iq2xs" : (kq == 23 ? "iq2s" : (kq == 34 ? "iq3xxs" : "iq3s"))) //! grid entries per format: the compact plane's byte count @@ -1911,9 +1919,14 @@ def private vbmi_lookup(var te : TileEmit; var tbl : LLVMOpaqueValue?; entries : var depth = 0 while (length(level) > 1) { var sel = depth == 0 ? high : masks[depth - 1] - var nxt <- [for (j in range(length(level) / 2)); LLVMBuildSelect(b, sel, level[2 * j + 1], level[2 * j], "{name}.s{depth}_{j}")] + var nxt : array + for (j in range(length(level) / 2)) { + nxt |> push(LLVMBuildSelect(b, sel, level[2 * j + 1], level[2 * j], "{name}.s{depth}_{j}")) + } level |> clear() - level <- nxt + for (v in nxt) { + level |> push(v) + } depth++ } return level[0] @@ -1940,13 +1953,13 @@ def private vbmi_col(var te : TileEmit; var wbP : LLVMOpaqueValue?; col, qd : in } //! the row's qh byte for this block (packed byte position 64 + blk: column 16 + blk/4, byte blk%4) broadcast -//! over its four lanes, then bit `bitOf(lane%4)` as a lane mask - the iq3s / iq2s index bits above the qs byte -def private vbmi_qh_mask(var te : TileEmit; var qhCol : LLVMOpaqueValue?; blk : int; bitOf : block<(p : int) : int>; name : string) : LLVMOpaqueValue? { +//! over its four lanes, then bit `bit0 + stride * (lane % 4)` as a lane mask - the iq3s / iq2s index bits above the qs byte +def private vbmi_qh_mask(var te : TileEmit; var qhCol : LLVMOpaqueValue?; blk, bit0, stride : int; name : string) : LLVMOpaqueValue? { let b = te.builder let w8 = te.width / 8 let idx <- [for (k in range(w8)); (k / 4) * 4 + blk % 4] var bytes = LLVMBuildShuffleVector(b, te.types, qhCol, qhCol, idx, "{name}.q") - var bits <- [for (k in range(w8)); LLVMConstInt(te.types.t_int8, uint64(1 << invoke(bitOf, k % 4)), 0)] + var bits <- [for (k in range(w8)); LLVMConstInt(te.types.t_int8, uint64(1 << (bit0 + stride * (k % 4))), 0)] var bitsv = LLVMConstVector(array_data_ptr(bits), uint(w8)) return LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntNE, LLVMBuildAnd(b, bytes, bitsv, ""), LLVMConstNull(te.vwi8), name) } @@ -1985,12 +1998,12 @@ def private emit_grid_rows_blk_vbmi(var te : TileEmit; var wbP, xb : LLVMOpaqueV } elif (te.kq == 33) { idx = vbmi_col(te, wbP, 2 * blk + c, qd, "gi{blk}_{c}") var qh = vbmi_col(te, wbP, 16 + blk / 4, qd, "qh{blk}") - masks |> push(vbmi_qh_mask(te, qh, blk, $(p : int) : int => 4 * c + p, "qm{blk}_{c}")) + masks |> push(vbmi_qh_mask(te, qh, blk, 4 * c, 1, "qm{blk}_{c}")) } elif (te.kq == 23) { idx = vbmi_col(te, wbP, blk, qd, "gi{blk}") var qh = vbmi_col(te, wbP, 16 + blk / 4, qd, "qh{blk}") - masks |> push(vbmi_qh_mask(te, qh, blk, $(p : int) : int => 2 * p, "qm{blk}_0")) - masks |> push(vbmi_qh_mask(te, qh, blk, $(p : int) : int => 2 * p + 1, "qm{blk}_1")) + masks |> push(vbmi_qh_mask(te, qh, blk, 0, 2, "qm{blk}_0")) + masks |> push(vbmi_qh_mask(te, qh, blk, 1, 2, "qm{blk}_1")) sgn = vbmi_col(te, wbP, 8 + blk, qd, "sg{blk}") } else { // 24: u16 words - columns 2blk (l 0, 1) and 2blk+1 (l 2, 3), lane r*4+l takes word l's low then high byte var ca = vbmi_col(te, wbP, 2 * blk, qd, "wa{blk}") From e42c38652189607f9edb4901a4ff0d04ed36f697 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 13:21:40 -0700 Subject: [PATCH 087/123] kernel parity plan: the cold-start crash bisected - vbmi perm-row presence, whole-module effect, cold+normal+team only --- plans/kernel_parity_pass.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 66734a7e2d..024e804281 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -866,5 +866,14 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. - 2026-09-01: cold-start SIGSEGV at 0xc0 right after "Library linked - ok" (normal mode, 16-lane team k6 d=32768, Intel; reproduced 3x cold, never warm, earlier bootstrap ladders passed) - a post-link rebind bug to chase before the PR. +- 2026-09-01: the cold-start SIGSEGV bisected. It needs only the PRESENCE of the five grid="vbmi" + tune_perm rows in dasllama_math_gen.das - not the emitter body, globals, or intrinsic decls (zen4 B1/B2/B3 + disable each in turn, all still crash), and not the startup race (zen4 T1: fingerprint patched so nothing + races, rows present -> crash; T2: rows removed, k6's own row made to race -> clean). It fires only cold + + normal mode + --team; warm run 2 and every tune-mode race are clean, and the crashing format is k6, not a + grid format - so it is a whole-module codegen/teardown effect of the extra generated variants (backtrace + LLVMContextImpl::~LLVMContextImpl, fault at a small offset), not the lattice IR. Correctness is unaffected + (TEST bit-exact, all races produced numbers). A focused C++ session owns the fix; the parity numbers come + from warm/tune runs meanwhile. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From ae2a881a9925547a94943a3c21f9f4cf8cb788f2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 14:06:09 -0700 Subject: [PATCH 088/123] vbmi lattice: one grid table per helper frame - five by-value tables in one das frame (~17 KB) overflowed the codegen context's stack into the heap, the cold-start SIGSEGV in LLVMContextDispose --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 69 +++++++++++++------ 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 1c57107fb5..3fde981979 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1800,31 +1800,19 @@ def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { var p1 : array p0 |> reserve(entries) p1 |> reserve(entries) - if (te.kq == 34) { // an iq3 entry is one word of 4 weights: weights 0-1 and 2-3 per plane - let g = iq3xxs_grid() - for (e in range(entries)) { - vbmi_push_iq3(te, p0, p1, g[e], alpha) - } + // one grid table per helper frame: a das frame holds every local of its function, and five by-value grid + // tables in one frame (~17 KB) overflowed the codegen context's stack into the heap (cold-start SIGSEGV + // in LLVMContextDispose); the existing global builders hold one 8 KB table each, and so do these + if (te.kq == 34) { + vbmi_planes_iq3xxs(te, p0, p1, alpha) } elif (te.kq == 33) { - let g = iq3s_grid() - for (e in range(entries)) { - vbmi_push_iq3(te, p0, p1, g[e], alpha) - } - } elif (te.kq == 25) { // an iq2 entry is two words: weights 0-3, 4-7 - one plane each - let g = iq2xxs_grid2() - for (e in range(entries)) { - vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) - } + vbmi_planes_iq3s(te, p0, p1, alpha) + } elif (te.kq == 25) { + vbmi_planes_iq2xxs(te, p0, p1, alpha) } elif (te.kq == 24) { - let g = iq2xs_grid2() - for (e in range(entries)) { - vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) - } + vbmi_planes_iq2xs(te, p0, p1, alpha) } else { - let g = iq2s_grid2() - for (e in range(entries)) { - vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) - } + vbmi_planes_iq2s(te, p0, p1, alpha) } te.vb_plane[0] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p0", p0) te.vb_plane[1] = vbmi_const_bytes(te, mod, "dasllama.vbmi.{tag}.p1", p1) @@ -1845,6 +1833,43 @@ def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { } } +//! an iq3 entry is one word of 4 weights: weights 0-1 and 2-3 per plane +def private vbmi_planes_iq3xxs(var te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq3xxs_grid() + for (e in range(256)) { + vbmi_push_iq3(te, p0, p1, g[e], alpha) + } +} + +def private vbmi_planes_iq3s(var te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq3s_grid() + for (e in range(512)) { + vbmi_push_iq3(te, p0, p1, g[e], alpha) + } +} + +//! an iq2 entry is two words: weights 0-3, 4-7 - one plane each +def private vbmi_planes_iq2xxs(var te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq2xxs_grid2() + for (e in range(256)) { + vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) + } +} + +def private vbmi_planes_iq2xs(var te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq2xs_grid2() + for (e in range(512)) { + vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) + } +} + +def private vbmi_planes_iq2s(var te : TileEmit; var p0, p1 : array; alpha : array) { + let g = iq2s_grid2() + for (e in range(1024)) { + vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) + } +} + def private vbmi_push_iq3(var te : TileEmit; var p0, p1 : array; word : uint; alpha : array) { p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(word, alpha, 3, 2)), 0)) p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(word >> 16u, alpha, 3, 2)), 0)) From c31d15848b832825fe562ab85c09f801352994f2 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 14:09:36 -0700 Subject: [PATCH 089/123] llvm_boost: LLVMBuildGEP2's inbounds goes through LLVMBuildInBoundsGEP2 - the post-hoc LLVMSetIsInBounds corrupted folded constant GEPs A GEP over a global with a constant index folds into a ConstantExpr; LLVMSetIsInBounds casts its operand to GetElementPtrInst, so on a ConstantExpr it wrote a flag into the constant's memory. Every emitter GEP until the vbmi lattice had a variable index or a parameter base, so nothing folded; the lattice's 64-byte table-part loads off constant globals were the first, and every cold normal-mode start on a VBMI box then died in LLVMContextDispose (glibc heap consolidation on the corrupted chunk; zen4 deterministic, Intel 3 of 4). LLVM_JIT_CODEGEN_VERSION 0x68. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLVM/daslib/llvm_boost.das | 8 +++++--- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/dasLLVM/daslib/llvm_boost.das b/modules/dasLLVM/daslib/llvm_boost.das index 653232fc0b..ba0df73292 100644 --- a/modules/dasLLVM/daslib/llvm_boost.das +++ b/modules/dasLLVM/daslib/llvm_boost.das @@ -307,12 +307,14 @@ def LLVMAddIncoming(phi : LLVMOpaqueValue?; var phi_vals : array Date: Tue, 1 Sep 2026 14:11:15 -0700 Subject: [PATCH 090/123] kernel parity plan: the cold-start crash found and fixed - the GEP wrapper's inbounds flag on folded constants --- plans/kernel_parity_pass.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 024e804281..476ab0f578 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -875,5 +875,14 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. LLVMContextImpl::~LLVMContextImpl, fault at a small offset), not the lattice IR. Correctness is unaffected (TEST bit-exact, all races produced numbers). A focused C++ session owns the fix; the parity numbers come from warm/tune runs meanwhile. +- 2026-09-01: the cold-start crash FOUND and FIXED (17e664e7f). Cuts on zen4: L1 (permi2b calls, no table + loads) clean vs L2 (table loads, no permi2b) crash pinned the lattice's 64-byte table-part loads; the core + showed glibc's malloc_consolidate under ~LLVMContextImpl - heap corruption at compile time. Cause: the + llvm_boost LLVMBuildGEP2 wrapper's `inbounds = true` default called LLVMSetIsInBounds on the result, and a + GEP over a global with a constant index folds into a ConstantExpr - the cast to GetElementPtrInst wrote a + flag into the constant's memory. Every emitter GEP before the lattice had a variable index or a parameter + base, so nothing ever folded. The wrapper now builds inbounds GEPs through LLVMBuildInBoundsGEP2. Cold + SOLO and cold 16-lane team both pass on zen4. Dead ends on the way: startup race path (T1/T2), frame size + (five by-value grid tables), constant folding of the loads, split partitions, lane count. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From d4a5876c473aa43f40a68e1e46872d02108f4a10 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 14:27:45 -0700 Subject: [PATCH 091/123] kernel_ladder.sh: KL_MODULE_CACHE passes -module-cache to the bench start - a cold dasLLAMA compile is minutes per row otherwise --- modules/dasLLAMA/harness/kernel_ladder.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh index 705d0edfb7..d8a9bf03b2 100644 --- a/modules/dasLLAMA/harness/kernel_ladder.sh +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -11,6 +11,8 @@ # Windows - never the global DASLANG, which names another tree's binary on dev boxes) # LCPP_TBO test-backend-ops (default: $HOME/Work/llama.cpp/build-clean-cpu/bin/Release/test-backend-ops.exe) # NTOK prefill tokens (default 512; 0 = decode rows only) +# KL_MODULE_CACHE a -module-cache path for the bench start (the AST cache: a cold dasLLAMA compile is +# minutes; keep it outside .jitted_scripts, which measurement scripts wipe) # ROUNDS interleaved rounds per row (default 5) # TEAM lanes for the decode rows: the bench dispatches the GEMV the engine's way (--team, # DAS_JOBQUE_THREADS=N) and the reference runs GGML_BENCH_THREADS=N. Default: the box's @@ -55,7 +57,9 @@ echo "# kernel ladder $(date +%F) box=$(hostname) daslang=$DASLANG ref=$TBO [ -x "$TBO" ] || { echo "kernel_ladder: no test-backend-ops at '$TBO' (set LCPP_TBO)" >&2; exit 1; } grep -q GGML_BENCH_THREADS "$TBO" || { echo "kernel_ladder: '$TBO' lacks the GGML_BENCH_THREADS define - it would run every core and the ratio column would lie (harness README)" >&2; exit 1; } set +e -DAS_TUNE_MODE=normal DAS_JOBQUE_THREADS=$THREADS "$DASLANG" -jit "$ROOT/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das" \ +MC_ARGS="" +if [ -n "${KL_MODULE_CACHE:-}" ]; then mkdir -p "$(dirname "$KL_MODULE_CACHE")"; MC_ARGS="-module-cache $KL_MODULE_CACHE"; fi +DAS_TUNE_MODE=normal DAS_JOBQUE_THREADS=$THREADS "$DASLANG" -jit $MC_ARGS "$ROOT/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das" \ -- --fmt "$FMTS" --d "$ROWS" --ntok "$NTOK" --rounds "$ROUNDS" --tsv --base-align 4096 $TEAM_ARGS > "$WORK/ours.raw" 2> "$WORK/ours.err" rc=$? set -e From b4d4918a343907b69e108794aa39b33a7974a671 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 14:33:57 -0700 Subject: [PATCH 092/123] tune profile x86-amx: re-minted on the lattice emitter (Granite Rapids c8i) - the five grid gemv seats take grid_vbmi 60 entries, TEST 103/103 earlier, provenance ok/ok, fingerprint with avx512vbmi. Ladders after it: every row at or past 0.95 of llama.cpp on one thread and 16 lanes except k6 at 16 lanes (0.68 - the normal-mode stall gap, open). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../defaults/x86-amx.tune-defaults.json | 33 ++++++++++++------- plans/kernel_parity_pass.md | 8 +++++ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json index f9b5ac9391..076428a928 100644 --- a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json @@ -1,7 +1,10 @@ { "kernels" : { + "iq4xsq8_gemv_gen" : "dot_vpdpbusd_width256_mr16", "add_inplace" : "vec16", + "iq2xsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "cvt_f32_to_f16" : "vec16", + "iq2sq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "rope_scaled_neox_tab" : "vec16", "q51q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "softmax" : "vec8_u2", @@ -14,10 +17,10 @@ "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "quantize_q8kv_row" : "plain", "axpy_f16" : "vec16_u2", - "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "q40q8_tile_gen" : "dot_vpdpbusd_width256_mr16", "axpy_tq4kv" : "vec8_u2", "cvt_tq4kv_to_f32" : "vec8_u2", - "axpy" : "vec16", + "axpy" : "vec16_u2", "dot_q8q8kv" : "vec16_u2", "dot_mx4q8" : "u2", "softmax_sink" : "vec8_u2", @@ -29,21 +32,29 @@ "cvt_q8kv_to_f32" : "vec16_u2", "axpy_q8kv" : "vec16_u2", "dot_q8q8_f16s" : "vec16", + "iq2xxsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", - "quantize_q8_0_into_ptr" : "plain", - "k4q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k5q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "k4q8_tile_gen" : "dot_vpdpbusd_width256_mr16", "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k6q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", "k3q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k3q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "iq4nlq8_gemv_gen" : "dot_vpdpbusd_width256_mr16", "k2q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k2q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "quantize_q8_0_into_ptr" : "plain", "gemm_f32_uk_4x16" : "u2", "dot_q51e" : "vec16", "dot_f16" : "vec16", "cvt_f16_to_f32" : "vec16", - "dot" : "vec8_u2", + "dot" : "vec16_u2", "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq3xxsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "dot_q8tq4kv" : "vec16", + "iq3sq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "scale_inplace" : "vec16", "dot_q4" : "vec8_u2", "quantize_tq4kv_row" : "plain", @@ -52,18 +63,18 @@ }, "provenance" : { "validation" : "ok", - "noise_probes" : "start cv 0.08%; mid1 cv 0.05%; mid2 cv 0.14%; end cv 0.05%", + "noise_probes" : "start cv 0.41%; mid1 cv 0.04%; mid2 cv 0.08%; end cv 0.20%", "platform" : "linux", - "noise_floor_cv_pct" : "0.14", - "features" : "avx2;f16c;fma;sse4.2;avxvnni;avx512f;avx512bw;avx512vl;avx512vnni;amx-tile;amx-int8", + "noise_floor_cv_pct" : "0.41", + "features" : "avx2;f16c;fma;sse4.2;avxvnni;avx512f;avx512bw;avx512vl;avx512vnni;avx512vbmi;amx-tile;amx-int8", "class" : "x86-amx", - "written" : "2026-09-01T17:51:26.103Z", - "validation_demoted" : "0", + "written" : "2026-09-01T21:21:48.426Z", + "validation_demoted" : "1", "mode" : "normal", "dasllama_version" : "15", "origin" : "profile", "noise" : "ok", "arch" : "x86_64", - "validation_max_drift_pct" : "4.97" + "validation_max_drift_pct" : "5.13" } } \ No newline at end of file diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 476ab0f578..bcafa8fccc 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -884,5 +884,13 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. base, so nothing ever folded. The wrapper now builds inbounds GEPs through LLVMBuildInBoundsGEP2. Cold SOLO and cold 16-lane team both pass on zen4. Dead ends on the way: startup race path (T1/T2), frame size (five by-value grid tables), constant folding of the loads, split partitions, lane count. +- 2026-09-01: Intel re-mint on the lattice emitter (x86-amx, 60 entries, provenance ok/ok, fingerprint carries + avx512vbmi): the gemv seats took grid_vbmi for all five grid families and 256_mr16 for iq4xs. The goal + ladders after it: one thread every row >= 1.18 (k6 1.19, iq2s 1.18, iq3s 2.19, iq3xxs 2.04, iq2xs 1.56, + iq2xxs 1.65); 16 lanes every row >= 0.95 (q8 exactly 0.95, iq3s 1.95, iq3xxs 2.48, iq2s 1.19, iq2xs 1.65, + iq2xxs 1.65) EXCEPT k6 0.68 (5346 vs 3635 us) - the normal-mode stall gap, Intel's last open row. +- 2026-09-01: cold-start cost is LLVM, not the das compile: the mint's JIT line reads optimize 60 s + emit+link + 72 s for the whole dasLLAMA module - the lattice bodies are fully unrolled over the 8 blocks (a called block + loop would cut it); -module-cache trims the das side only. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 369d552a3fe2bfa60b6001aed91a1e5d4b635641 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 14:36:55 -0700 Subject: [PATCH 093/123] tune profile x86-vnni512: re-minted on the lattice emitter (zen4 c7a) - the five grid gemv seats take grid_vbmi; the v4 goal ladders 56 entries, provenance ok/ok, fingerprint with avx512vbmi. Ladders after it (zen4 / Intel): every one-thread row past the reference; every grid row at 16 lanes 1.19-2.48x. Still under 0.95 at 16 lanes: zen4 q8 0.89, iq4xs 0.92, q51 0.94; Intel k6 0.68. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../defaults/x86-vnni512.tune-defaults.json | 26 ++++++++++++------- plans/kernel_parity_pass.md | 25 ++++++++++++++++++ 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json index 7c17a09fd7..bcb0cbe008 100644 --- a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json @@ -1,9 +1,11 @@ { "kernels" : { "add_inplace" : "vec8_u2", + "iq2xsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "cvt_f32_to_f16" : "plain", + "iq2sq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "rope_scaled_neox_tab" : "plain", - "q51q8_tile_gen" : "dot_vpdpbusd_width256_mr8", + "q51q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "softmax" : "vec8_u2", "mul_inplace" : "vec8_u2", "quantize_q8_0_bs_into_ptr" : "u2", @@ -13,7 +15,7 @@ "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "quantize_q8kv_row" : "plain", - "axpy_f16" : "vec8_u2", + "axpy_f16" : "plain", "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "axpy_tq4kv" : "vec8_u2", "cvt_tq4kv_to_f32" : "vec8_u2", @@ -27,15 +29,17 @@ "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "add_scale_inplace" : "vec8_u2", "cvt_q8kv_to_f32" : "vec8_u2", - "axpy_q8kv" : "vec8_u2", + "axpy_q8kv" : "plain", "dot_q8q8_f16s" : "vec16", + "iq2xxsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", "k3q8_gemv_gen" : "dot_vpdpbusd_width256_mr8", - "k4q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k4q8_tile_gen" : "dot_vpdpbusd_width256_mr16", "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "k3q8_tile_gen" : "dot_maddubs_width256_mr8", - "k2q8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "k2q8_tile_gen" : "dot_vpdpbusd_width256_mr16", + "k2q8_gemv_gen" : "dot_vpdpbusd_width512_mr16", "quantize_q8_0_into_ptr" : "plain", "gemm_f32_uk_4x16" : "u2", "dot_q51e" : "vec16", @@ -43,8 +47,10 @@ "cvt_f16_to_f32" : "vec8_u2", "dot" : "vec16", "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq3xxsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "dot_q8tq4kv" : "vec16", + "iq3sq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "scale_inplace" : "vec8_u2", "dot_q4" : "vec8_u2", "quantize_tq4kv_row" : "plain", @@ -53,18 +59,18 @@ }, "provenance" : { "validation" : "ok", - "noise_probes" : "start cv 0.15%; mid1 cv 0.08%; mid2 cv 0.18%; end cv 0.08%", + "noise_probes" : "start cv 0.10%; mid1 cv 0.05%; mid2 cv 0.36%; end cv 0.05%", "platform" : "linux", - "noise_floor_cv_pct" : "0.18", - "features" : "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni", + "noise_floor_cv_pct" : "0.36", + "features" : "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni;avx512vbmi", "class" : "x86-vnni512", - "written" : "2026-09-01T17:52:25.911Z", + "written" : "2026-09-01T21:22:56.997Z", "validation_demoted" : "1", "mode" : "normal", "dasllama_version" : "15", "origin" : "profile", "noise" : "ok", "arch" : "x86_64", - "validation_max_drift_pct" : "6.18" + "validation_max_drift_pct" : "2.85" } } \ No newline at end of file diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index bcafa8fccc..59011b548b 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -493,6 +493,31 @@ one thread shows it as best 1831 / median 2925 inside ONE process, so the slow m flips between rounds, not between processes) and iq2s 0.90 (0.93 one thread). No gemv seat was written on this box, so the k6 gemv still rides the 512 tile crown here. +### zen4 v4 and Intel v4 - the goal ladders on the lattice re-mints (2026-09-01; SOLO one thread d=4096, TEAM 16 lanes d=32768; ratio = reference / ours) + +| fmt | zen4 gemv 1T | zen4 tile 1T | zen4 gemv 16L | Intel gemv 1T | Intel tile 1T | Intel gemv 16L | +|---|---|---|---|---|---|---| +| q8 | 1.81 | 3.01 | 0.89 | 1.25 | 4.00 | 0.95 | +| k4 | 2.45 | 2.19 | 1.05 | 1.52 | 3.88 | 1.47 | +| k5 | 1.26 | 6.33 | 1.00 | 1.69 | 6.78 | 1.48 | +| k6 | 1.10 | 3.43 | 1.01 | 1.19 | 4.14 | 0.68 | +| q40 | 4.31 | 3.39 | 1.00 | 2.07 | 3.84 | 2.55 | +| q51 | 2.84 | 5.03 | 0.94 | 2.51 | 7.01 | 1.84 | +| iq4xs | 2.13 | 4.03 | 0.92 | 2.14 | 4.26 | 2.32 | +| k3 | 2.18 | 2.75 | 0.99 | 1.85 | 4.11 | 1.01 | +| iq3s | 2.20 | 11.87 | 1.23 | 2.19 | 14.07 | 1.95 | +| iq3xxs | 2.18 | 10.22 | 1.25 | 2.04 | 10.39 | 2.48 | +| iq4nl | 2.78 | 2.20 | 0.95 | 2.42 | 2.73 | 3.03 | +| k2 | 2.06 | 1.49 | 1.14 | 2.28 | 2.66 | 1.70 | +| iq2s | 1.33 | 6.70 | 1.21 | 1.18 | 6.90 | 1.19 | +| iq2xs | 1.43 | 6.50 | 1.31 | 1.56 | 8.08 | 1.65 | +| iq2xxs | 1.79 | 7.64 | 1.44 | 1.65 | 7.26 | 1.65 | +| mx4 | 2.60 | 4.57 | 1.00 | 2.52 | 4.80 | 2.60 | + +Reading: every one-thread row on both boxes is past the reference (zen4 >= 1.10, Intel >= 1.18); every grid row at 16 lanes is 1.19-2.48x. +Under 0.95 at 16 lanes: zen4 q8 0.89, iq4xs 0.92, q51 0.94 (the DRAM-bound rows; iq4nl exactly 0.95) and Intel k6 0.68 (the normal-mode stall gap). +Both minted profiles committed; the gemv seats took grid_vbmi for all five grid families on both boxes. + ## 3. Research memos (read before touching the kernels) - `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, From fe439d0b9ea2ea09008844dabc8d0363db833a2f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 14:55:40 -0700 Subject: [PATCH 094/123] kq_kernel_bench: a '# ysum' line per gemv row under --tsv - modes and seats must agree on the answer before their times are compared --- modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 01d8eb84ec..24a6215936 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -414,6 +414,16 @@ def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va invoke(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) } } + if (cfg.tsv) { // the row's answer, one solo call each: modes and seats must agree here before their times are compared + for (row in range(length(fns))) { + invoke(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) + var acc = 0.0lf + for (i in range64(d)) { + acc += double(yp[i]) + } + print("# ysum\t{name}\t{stats[row].perm}\t{acc}\t{yp[0]}\t{yp[d - 1l]}\n") + } + } } for (st in stats) { emit_row(rows, name, "gemv", st, n, d, 1l, nsb, plane_bytes) From 0d6cf5d849afb808d680aca171f2936a2b4c8b1f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 15:00:59 -0700 Subject: [PATCH 095/123] kernel parity plan: the Intel k6 mode gap vanished with the rebase; zen4's 16-lane rows are grain and MLP, not mode --- plans/kernel_parity_pass.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 59011b548b..41c4bf2f77 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -917,5 +917,18 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. - 2026-09-01: cold-start cost is LLVM, not the das compile: the mint's JIT line reads optimize 60 s + emit+link 72 s for the whole dasLLAMA module - the lattice bodies are fully unrolled over the 8 blocks (a called block loop would cut it); -module-cache trims the das side only. +- 2026-09-01: the Intel k6 16-lane mode gap VANISHED after the rebase onto master (d76d86fe0 + binary + rebuild): normal 3434 us vs tune 3287 / 3119 (was 5323 vs 3698 for the identical 512 body); answers + bit-identical across modes (the bench's new '# ysum' line). Two facts changed at once on the pre-rebase + tree: the stamped gemv companion carried the TILE's 512_mr16 body although the profile named the 256_mr16 + seat (the disassembly: outlined 491-insn EVEX body called from the das thunk), and the same 512 body ran + 1.44x slower in normal mode than as a tune-mode clone - both gone now (normal stamps the VEX 256 seat). + Not isolated to a master commit; if the gap ever returns, bisect llvm_tune.das between + bbatkin/kernel-parity-pre-rebase and the rebased tip. Ruled out along the way: alignment/arena phase, + THP, L3 residency, dispatch grain, thread placement, per-call interop cost, outlined-vs-inlined body. +- 2026-09-01: zen4 16-lane rows are NOT a mode gap (tune == normal: q8 6715 / 6733, iq4xs 3077 / 3008). Grain + matters for balance: at --chunks-per-lane 16 iq4xs 2978 (0.96 of 2856) and q51 4132 (0.99 of 4093) with + balanced lanes; q8 stays ~6700 at every grain (0.90) with every lane busy the whole wall - 70 GB/s against + the reference's 82 on q8 while our k4 streams at 91 on the same box: memory-level parallelism, not bandwidth. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 8e71b524263424df404124d0de9f5f8ed660c727 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 15:17:16 -0700 Subject: [PATCH 096/123] tune harness: the gemv seat races at the engine's decode shape; gemv chunks per lane default 16 The streamed fixture (n=2048, d=8192) fits L3 on server parts, so every gemv seat was decided cache-resident - on Granite Rapids that crowned k3's 256_mr16 seat (26% ahead there) which loses 7% at 16 lanes against the 512 body. The seat now races the tile winner's same-layout top two over a d=32768 x n=14336 plane through the engine's own splitter on every lane (run_kq_gemv_team; the one-lane box keeps the streamed time); the sidecar row carries team_us. gemv_chunks_per_lane 8 -> 16: zen4 at 16 lanes, d=32768, iq4xs 3108 -> 2978 us and q51 4192 -> 4132 with balanced lanes; the grain only changes past 64*waves*lanes rows. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/dasllama/dasllama_math.das | 6 +- modules/dasLLAMA/harness/gen_tune_probe.das | 69 ++++++++++++++++++++- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 868c319512..4296d25035 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -500,8 +500,10 @@ def public get_matmul_min_chunk_rows_gemv() : int { return g_matmul_min_chunk_ro // GEMV tail: the one-chunk-per-lane equal split leaves the join waiting out the SLOWEST lane's // whole chunk (median lane idles 14-24% of every GEMV stage, per-lane trace 3990X). >1 publishes -// wave-aligned chunks per lane so finished lanes keep serving; default 8; 1 = historical equal split. -var g_gemv_chunks_per_lane = 8 +// wave-aligned chunks per lane so finished lanes keep serving; default 16 (zen4 16 lanes, d=32768: iq4xs +// 3108 -> 2978 us, q51 4192 -> 4132 against 8, lanes balanced; the grain only bites past 64*waves*lanes rows); +// 1 = historical equal split. +var g_gemv_chunks_per_lane = 16 def public set_gemv_chunks_per_lane(v : int) { g_gemv_chunks_per_lane = max(v, 1) } def public get_gemv_chunks_per_lane() : int { return g_gemv_chunks_per_lane } diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 86e8b30cff..60dcd12c68 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -32,6 +32,8 @@ options _dasllama_internal = true // first). require dasllama/dasllama_math +require dasllama/dasllama_par // maybe_parallel_for_indexed - the gemv seat races at the engine's decode shape +require daslib/jobque_boost require dasllama/dasllama_math_default require dasllama/dasllama_math_aarch64_neon require dasllama/dasllama_math_gen @@ -1016,6 +1018,23 @@ def run_kq_gemv(gemv; fx : KqFixture; kq : array; ks : array; mr : } } +//! the engine's decode dispatch over the fixture: the engine's own splitter and every lane streaming its share +def run_kq_gemv_team(gemv; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { + let d = (fx.d / mr) * mr + let n = fx.n + unsafe { + var yp = addr(y[0]) + let kqp = addr(kq[0]) + let ksp = addr(ks[0]) + let xqp = addr(fx.xq[0]) + let xsp = addr(fx.xs[0]) + let xbsp = addr(fx.xbs[0]) + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { + invoke(gemv, yp, kqp, ksp, xqp, xsp, xbsp, n, int64(rb), int64(re)) + } + } +} + // k6's signed sub-scales cancel within a row, so element-relative tolerance is too strict at // production shapes (intermediate terms are large, the sum is small) — gate against the // fixture's magnitude instead; a real generator bug (wrong stride/lane/scale) is orders louder @@ -1407,10 +1426,56 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge } var tnames <- [for (t in tvs); t._0] winner = gen_sticky(kq_tile_entry(fmt), tnames, best, winner) - gemv_out = kq_gemv_seat(fmt, winner, tnames, vmr, best, gbest) + // the seat is decided at the engine's decode shape - a DRAM-bound plane (the streamed fixture fits L3 on + // server parts) streamed by every lane through the engine's splitter; the one-lane box keeps the streamed time + var tbest : array + tbest |> resize(nv) + for (i in range(nv)) { + tbest[i] = gbest[i] + } + if (get_total_hw_jobs() > 1) { + var ktfx <- build_kq_fixture(fmt, 14336l, 32768l, 1l) + var tbuf : KqBuf + tbuf.q := ktfx.kq + tbuf.s := ktfx.ks + unsafe { + repack_kq_grp_fmt(fmt, addr(tbuf.q[0]), addr(tbuf.s[0]), ktfx.n, ktfx.d, vmr[winner]) + } + var kyt : array + kyt |> resize(ktfx.d) + for (i in range(nv)) { + tbest[i] = 1.0e30lf + } + with_job_que() { + for (r in range(5)) { + for (vi in range(nv)) { + continue if (vmr[vi] != vmr[winner] || best[vi] >= 1.0e29lf) + for (_w in range(r == 0 ? 3 : 0)) { + run_kq_gemv_team(gtab[tvs[vi]._0], ktfx, tbuf.q, tbuf.s, vmr[vi], kyt) + } + let t0 = ref_time_ticks() + run_kq_gemv_team(gtab[tvs[vi]._0], ktfx, tbuf.q, tbuf.s, vmr[vi], kyt) + let us = double(get_time_usec(t0)) + if (us < tbest[vi]) { + tbest[vi] = us + } + } + } + } + for (vi in range(nv)) { + if (vmr[vi] == vmr[winner] && best[vi] < 1.0e29lf) { + tune_detail("k{fmt} {tvs[vi]._0}: decode at the engine shape {tbest[vi]} us ({get_total_hw_jobs()} lanes, d={ktfx.d} n={ktfx.n})\n") + } + } + delete kyt + delete tbuf + delete ktfx + } + gemv_out = kq_gemv_seat(fmt, winner, tnames, vmr, best, tbest) if (!empty(gemv_out)) { - rows_out["gemv_seat"] = JV((seat = gemv_out, tile_seat = tvs[winner]._0, gemv_us = gbest[winner])) + rows_out["gemv_seat"] = JV((seat = gemv_out, tile_seat = tvs[winner]._0, gemv_us = gbest[winner], team_us = tbest[winner])) } + delete tbest delete tnames let wname = tvs[winner]._0 delete keep From 7db15e626cbd1d1d7e2de9ebbf7e6d103ac8c03c Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 15:20:42 -0700 Subject: [PATCH 097/123] gemm emitter: a pf perm knob - software prefetch pf bytes ahead on the gemv's weight stream (q8 block); a pf=2048 lab row on the q8 grid zen4 at 16 lanes streams q8 at 70 GB/s against the reference's 82 while our k4 draws 91 on the same box, with every lane busy the whole wall: memory-level parallelism, not bandwidth. One llvm.prefetch per weight line, pf bytes ahead, gemv only (tokCount 1); the tile and the ARM legs decline pf. Lab row, unraced for the crown. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 35 +++++++++++++++++-- .../dasLLAMA/dasllama/dasllama_math_gen.das | 1 + 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 3fde981979..126754bf66 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -100,6 +100,9 @@ struct private TilePerm { // ldtilecfg/tilerelease — the cfg companion owns per-chunk config grid : string = "" // "vbmi": the grid formats' gemv decodes through the VBMI symbol lattice // (the row form; the tile and the planes are the plain row's) + pf : int = 0 // software prefetch distance in bytes along the gemv's weight stream (0 = none): + // one llvm.prefetch per weight line, pf bytes ahead - memory-level parallelism at + // many lanes, where the hardware prefetcher alone leaves DRAM bandwidth unused } def private parse_perm(gc : LlvmCodeCtx) : TilePerm { @@ -124,6 +127,8 @@ def private parse_perm(gc : LlvmCodeCtx) : TilePerm { if (lv is tInt) { p.latch = lv as tInt } let grv = find_arg(gc.ann.arguments, "grid") if (grv is tString) { p.grid = grv as tString } + let pfv = find_arg(gc.ann.arguments, "pf") + if (pfv is tInt) { p.pf = pfv as tInt } return p } @@ -140,7 +145,7 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint || (p.bias != 0 && p.bias != 128) || (p.pipe != 0 && p.pipe != 1) || (p.latch != 0 && p.latch != 1) - || (p.grid != "") + || (p.grid != "") || (p.pf != 0) || (!g_target_is_x64) || (!g_target_x64_amx) || (!g_target_x64_avx512vnni) // the companions' busd512 lattice + zmm fold || (!g_target_os_linux)) return true @@ -177,12 +182,13 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint || (p.gkstep != 1 && p.gkstep != 2 && p.gkstep != 4) || (p.mr < 4) || (p.pipe != 0) || (p.latch != 0) || (p.grid != "" && p.grid != "vbmi") + || (p.pf != 0 && p.pf != 1024 && p.pf != 2048 && p.pf != 4096) || (p.bias != 0 && (p.bias != 128 || p.dotPrim != "vpdpbusd"))) return true let rt = q8q8_repack_type(p.mr, p.bias, kgroup_of(p)) if (rt.interleave != p.mr || !rt.f32_scales) return true if (p.dotPrim == "smmla") { // the i8mm MMA leg: NEON widths, row-quad geometry, same q-reg budget as sdot, and the i8mm tier flag (M1 is dotprod-only) - if ((!g_target_is_aarch64) || (!g_target_arm64_i8mm) || (p.width != 128) || (p.grid != "") + if ((!g_target_is_aarch64) || (!g_target_arm64_i8mm) || (p.width != 128) || (p.grid != "") || (p.pf != 0) || (p.mr % 4 != 0) || (2 * p.mr + p.nrsplit * p.mr / 4 + 2 * p.nrsplit + 2 > 32)) return true let mod = LLVMGetGlobalParent(gc.impl) @@ -200,7 +206,7 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint } if (p.dotPrim == "sdot") { // the NEON leg (M2 shape): 128-bit q-regs, 4-row sdot lanes; mr>=8 budgets the FUSED-acc shape (kq shares one lo+hi acc there — half the mr4 formula's int-acc bank) - if ((!g_target_is_aarch64) || (p.width != 128) || (p.grid != "") + if ((!g_target_is_aarch64) || (p.width != 128) || (p.grid != "") || (p.pf != 0) || (p.mr % 4 != 0) || (p.mr >= 8 ? (p.mr + p.nrsplit * p.mr / 4 + 2 * p.nrsplit + 2 > 32) : (2 * p.mr + p.nrsplit * p.mr / 4 + 2 * p.nrsplit + 2 > 32))) return true @@ -259,6 +265,9 @@ struct private TileEmit { sign_code_pm1_g : LLVMOpaqueValue? // the row-group grid decode: ksigns code -> eight +-1 bytes, [128 x i64], parity bit folded in gridRows : bool // set while the row-group grid decode emits: the column dword read (a panel-form win) reverts to byte loads vbmi : bool // grid = "vbmi": the grid gemv decodes through the VBMI symbol lattice (row form) + pf : int // software prefetch distance along the weight stream (bytes; 0 = none) + prefetch_decl : LLVMOpaqueValue? // llvm.prefetch.p0 + prefetch_ty : LLVMOpaqueType? vb_plane : LLVMOpaqueValue? [2] // the grid's compact code planes, [256 x i8] each: the entry's low / high symbols vb_alpha : LLVMOpaqueValue? // the symbol alphabet as a vpshufb table (16 bytes per lane, [64 x i8]) vb_ksigns : LLVMOpaqueValue? // ksigns_iq2xs as [128 x i8] - one VPERMI2B turns 7-bit codes into sign bytes, parity in @@ -444,6 +453,14 @@ def private bias_acc_init(var te : TileEmit; var bi, xlo, xhi : LLVMOpaqueValue? // One 32-k block for tokens [tokBase, tokBase+tokCount): 8*rq shared weight vectors x tokens // lane-dots at full width, then the per-block scale fold — the exact op order of // dot_q8q8_laneq4x4_template, so fast-math contraction sees the same input across emitters. +//! llvm.prefetch(read, high locality, data) of the weight plane at byte offset `off` +def private prefetch_wg(var te : TileEmit; var off : LLVMOpaqueValue?) { + let b = te.builder + var p = LLVMBuildGEP2(b, te.types.t_int8, te.wg, off, "") + var args <- [p, te.types->ConstI32(0ul), te.types->ConstI32(3ul), te.types->ConstI32(1ul)] + LLVMBuildCall2(b, te.prefetch_ty, te.prefetch_decl, args, "") +} + def private emit_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { let b = te.builder let rq = te.rq @@ -477,6 +494,9 @@ def private emit_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLV var off = boff == 0 ? wb : LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(boff)), "") wv[kg * rq + qd] = load_vec(te, te.vwi8, te.wg, off, "wv{kg * rq + qd}") uw[kg * rq + qd] = abs_w(te, wv[kg * rq + qd], "uw{kg * rq + qd}") + if (te.pf != 0 && tokCount == 1) { // the gemv's weight stream: the same line, pf bytes ahead + prefetch_wg(te, LLVMBuildAdd(b, off, te.types->ConstI64(uint64(te.pf)), "")) + } } } } @@ -2542,6 +2562,15 @@ def private setup_tile_emit(var te : TileEmit; var gc : LlvmCodeCtx; p : TilePer var redArgTypes <- [te.vni32] te.reduce_ty = LLVMFunctionType(te.types.t_int32, redArgTypes) } + te.pf = p.pf + if (te.pf != 0) { + var pty = LLVMPointerType(te.types.t_int8, 0u) + var pfTypes <- [pty] + te.prefetch_decl = LLVMGetIntrinsicDeclaration(mod, LLVMLookupIntrinsicID("llvm.prefetch"), pfTypes) + if (te.prefetch_decl == null) return false + var pfArgTypes <- [pty, te.types.t_int32, te.types.t_int32, te.types.t_int32] + te.prefetch_ty = LLVMFunctionType(te.types.t_void, pfArgTypes) + } te.vbmi = p.grid == "vbmi" if (te.vbmi) { var b3Types <- [te.vwi8, te.vwi8, te.vwi8] diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index add6bb0b7a..cbaf47f98e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -1189,6 +1189,7 @@ def q8q8_tile_s16_gen(var yp : float?; wg : int8 const?; sg : uint16 const?; xqp tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, gkstep = 2, bias = 128, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, pf = 2048, requires = "avx512vnni,avx512bw"), // the crown with a 2 KB software prefetch on the gemv stream (lab row) tune_perm(dot = "vpdpbssd", width = 256, mr = 8, kstep = 2, requires = "avxvnniint8"), tune_perm(dot = "smmla", mr = 4, kstep = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 4, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 4, nrsplit = 2, requires = "i8mm"), From 8187672a28d35f0d0e513a41468841c6153518f8 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 15:39:06 -0700 Subject: [PATCH 098/123] tune harness: the engine-shape seat race tiles the streamed fixture (no reference rebuild) inside the live jobque; repack scratch reserves first The first cut called get_total_hw_jobs() before with_job_que() (the tuner child died: 'need to be in a with_job_que block', the sidecar came back with 34 entries and no seats) and built a fresh 32768 x 14336 fixture whose scalar reference gemv takes minutes per family. Now: the streamed fixture's master planes are tiled 48x straight into the repack buffer (393216 rows at n=2048, 200-660 MB, byte-identical row groups, x reused), repacked in place, raced through the engine's splitter on every lane. dasllama_repack's plane-size scratch copies reserve before resize - a bare resize past 64 MB trips max_unreserved_size. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/dasllama/dasllama_repack.das | 30 +++++++++ modules/dasLLAMA/harness/gen_tune_probe.das | 63 ++++++++++++------- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index b34b98833a..9ed108f008 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -56,7 +56,9 @@ def repack_q8q8_grp(var wp : int8?; var sp : float?; n, d, mr : int64; wbias : i let ncg = n / kgroup var tw : array var ts : array + tw |> reserve(d * n) // the plane-size scratch: reserve first, a bare resize past 64 MB trips max_unreserved_size tw |> resize(d * n) + ts |> reserve(d * nb) ts |> resize(d * nb) unsafe { var twp = addr(tw[0]) @@ -102,7 +104,9 @@ def repack_mx4_grp(var np : uint8?; var ep : uint8?; n, d, mr : int64) { let ng = d / mr var tn : array var te : array + tn |> reserve(d * nrow) tn |> resize(d * nrow) + te |> reserve(d * nbb) te |> resize(d * nbb) unsafe { var tnp = addr(tn[0]) @@ -144,7 +148,9 @@ def repack_q51_grp(var qp : uint8?; var sp : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * nb * 20l) tq |> resize(d * nb * 20l) + ts |> reserve(d * nb * 4l) ts |> resize(d * nb * 4l) unsafe { var tqp = addr(tq[0]) @@ -194,7 +200,9 @@ def repack_k4_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -240,7 +248,9 @@ def repack_k5_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -290,7 +300,9 @@ def repack_k6_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -349,7 +361,9 @@ def repack_q40_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -389,7 +403,9 @@ def repack_iq4xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -434,7 +450,9 @@ def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -490,7 +508,9 @@ def repack_iq3s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -535,7 +555,9 @@ def repack_k2_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -579,7 +601,9 @@ def repack_iq2s_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -623,7 +647,9 @@ def repack_iq2xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -667,7 +693,9 @@ def repack_iq2xxs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) @@ -760,7 +788,9 @@ def repack_iq3xxs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let ng = d / mr var tq : array var ts : array + tq |> reserve(d * qrow) tq |> resize(d * qrow) + ts |> reserve(d * srow) ts |> resize(d * srow) unsafe { var tqp = addr(tq[0]) diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index 60dcd12c68..d3ec602b1e 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1018,6 +1018,24 @@ def run_kq_gemv(gemv; fx : KqFixture; kq : array; ks : array; mr : } } +//! the streamed fixture's master planes repeated `times` over into a repack buffer: a DRAM-bound plane at the +//! same n (byte-identical row groups), x reused, no reference - a fresh build at that size would transcode +//! millions of superblocks and run a scalar reference gemv over them, minutes per family +def tile_kq_planes(fx : KqFixture; times : int64; var buf : KqBuf) { + let qrow = length(fx.kq) + let srow = length(fx.ks) + buf.q |> reserve(qrow * int(times)) + buf.q |> resize(qrow * int(times)) + buf.s |> reserve(srow * int(times)) + buf.s |> resize(srow * int(times)) + unsafe { + for (t in range64(times)) { + memcpy(addr(buf.q[int(t) * qrow]), addr(fx.kq[0]), qrow) + memcpy(addr(buf.s[int(t) * srow]), addr(fx.ks[0]), srow) + } + } +} + //! the engine's decode dispatch over the fixture: the engine's own splitter and every lane streaming its share def run_kq_gemv_team(gemv; fx : KqFixture; kq : array; ks : array; mr : int64; var y : array) { let d = (fx.d / mr) * mr @@ -1433,20 +1451,23 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge for (i in range(nv)) { tbest[i] = gbest[i] } - if (get_total_hw_jobs() > 1) { - var ktfx <- build_kq_fixture(fmt, 14336l, 32768l, 1l) - var tbuf : KqBuf - tbuf.q := ktfx.kq - tbuf.s := ktfx.ks - unsafe { - repack_kq_grp_fmt(fmt, addr(tbuf.q[0]), addr(tbuf.s[0]), ktfx.n, ktfx.d, vmr[winner]) - } - var kyt : array - kyt |> resize(ktfx.d) - for (i in range(nv)) { - tbest[i] = 1.0e30lf - } - with_job_que() { + with_job_que() { // the lane count and the team dispatch both need the live jobque + if (get_total_hw_jobs() > 1) { + let TILES = 48l // 393216 rows at n=2048: 200-660 MB per format, past any server L3 + var ktfx = KqFixture(fmt = kgfx.fmt, n = kgfx.n, d = kgfx.d * TILES, ntok = 1l, nsb = kgfx.nsb) + ktfx.xq := kgfx.xq + ktfx.xs := kgfx.xs + ktfx.xbs := kgfx.xbs + var tbuf : KqBuf + tile_kq_planes(kgfx, TILES, tbuf) + unsafe { + repack_kq_grp_fmt(fmt, addr(tbuf.q[0]), addr(tbuf.s[0]), ktfx.n, ktfx.d, vmr[winner]) + } + var kyt : array + kyt |> resize(ktfx.d) + for (i in range(nv)) { + tbest[i] = 1.0e30lf + } for (r in range(5)) { for (vi in range(nv)) { continue if (vmr[vi] != vmr[winner] || best[vi] >= 1.0e29lf) @@ -1461,15 +1482,15 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge } } } - } - for (vi in range(nv)) { - if (vmr[vi] == vmr[winner] && best[vi] < 1.0e29lf) { - tune_detail("k{fmt} {tvs[vi]._0}: decode at the engine shape {tbest[vi]} us ({get_total_hw_jobs()} lanes, d={ktfx.d} n={ktfx.n})\n") + for (vi in range(nv)) { + if (vmr[vi] == vmr[winner] && best[vi] < 1.0e29lf) { + tune_detail("k{fmt} {tvs[vi]._0}: decode at the engine shape {tbest[vi]} us ({get_total_hw_jobs()} lanes, d={ktfx.d} n={ktfx.n})\n") + } } + delete kyt + delete tbuf + delete ktfx } - delete kyt - delete tbuf - delete ktfx } gemv_out = kq_gemv_seat(fmt, winner, tnames, vmr, best, tbest) if (!empty(gemv_out)) { From 972d08e22f361c89d0da0c6a15a8a475e415088f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 15:41:57 -0700 Subject: [PATCH 099/123] kernel parity plan: x86 hybrid core detection follow-up (Apple-only today; goal boxes homogeneous) --- plans/kernel_parity_pass.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 41c4bf2f77..81d35ecf0c 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -930,5 +930,10 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. matters for balance: at --chunks-per-lane 16 iq4xs 2978 (0.96 of 2856) and q51 4132 (0.99 of 4093) with balanced lanes; q8 stays ~6700 at every grain (0.90) with every lane busy the whole wall - 70 GB/s against the reference's 82 on q8 while our k4 streams at 91 on the same box: memory-level parallelism, not bandwidth. +- 2026-09-01: FOLLOW-UP (Boris): x86 hybrid core detection. get_num_perf_cores / is_slow_tier_compute are Apple-only + (job_que.cpp), so on hybrid Intel clients (12th-15th gen) tg runs E-cores as team lanes and the join waits on + their last chunk; pp is fine (homogeneous ISA, extra compute). Detection: Linux /sys/devices/cpu_core/cpus vs + cpu_atom/cpus; Windows GetLogicalProcessorInformationEx EfficiencyClass. The darwin policy then applies as is. + Not a factor on the goal boxes (c7a EPYC 9R14 and c8i Xeon 6975P-C are homogeneous - verified, no cpu_atom). - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 14f72caddd2f0dbf96ccaa12ce8e26bddf989511 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 15:45:43 -0700 Subject: [PATCH 100/123] followup_general 66: x86 hybrid core detection for the lane policy (Apple-only today) --- modules/dasLLAMA/followup_general.md | 10 ++++++++++ plans/kernel_parity_pass.md | 7 ++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index c69f5c7341..1fba341751 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -833,3 +833,13 @@ once already. The tests checklist ledgers the residue; the fix is an in-process equivalent of `DASLLAMA_IMAGE=0` (`g_env_engine.image` is a `let` read at load), so such cells can run image-free instead of risking the purge. + +66. **x86 hybrid core detection for the lane policy.** `get_num_perf_cores` / `is_slow_tier_compute` + are Apple-only (`job_que.cpp`), so `dasllama_jobque_threads_cap` applies its split - every core + for prefill, perf cores for the decode lanes - on darwin alone. A hybrid Intel client (12th-15th + gen) runs its E-cores as decode team lanes and the join waits on their last chunk; prefill is + fine there (homogeneous ISA, extra compute). Detection is small: Linux counts + `/sys/devices/cpu_core/cpus` against `/sys/devices/cpu_atom/cpus` (hybrid Intel, kernel 5.13+), + Windows reads `EfficiencyClass` from `GetLogicalProcessorInformationEx(RelationProcessorCore)`; + the darwin branch then applies unchanged. No EC2 instance has such a part (server Xeons only); + a Hetzner EX44 (i5-13500, 6P+8E) or EX101 (i9-13900) rents by the hour for the check. diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 81d35ecf0c..ef64ebb4ba 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -930,10 +930,7 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. matters for balance: at --chunks-per-lane 16 iq4xs 2978 (0.96 of 2856) and q51 4132 (0.99 of 4093) with balanced lanes; q8 stays ~6700 at every grain (0.90) with every lane busy the whole wall - 70 GB/s against the reference's 82 on q8 while our k4 streams at 91 on the same box: memory-level parallelism, not bandwidth. -- 2026-09-01: FOLLOW-UP (Boris): x86 hybrid core detection. get_num_perf_cores / is_slow_tier_compute are Apple-only - (job_que.cpp), so on hybrid Intel clients (12th-15th gen) tg runs E-cores as team lanes and the join waits on - their last chunk; pp is fine (homogeneous ISA, extra compute). Detection: Linux /sys/devices/cpu_core/cpus vs - cpu_atom/cpus; Windows GetLogicalProcessorInformationEx EfficiencyClass. The darwin policy then applies as is. - Not a factor on the goal boxes (c7a EPYC 9R14 and c8i Xeon 6975P-C are homogeneous - verified, no cpu_atom). +- 2026-09-01: x86 hybrid core detection (Apple-only today) is LEDGERED as followup_general.md entry 66, not an arc + follow-up; the goal boxes are homogeneous (c7a EPYC 9R14, c8i Xeon 6975P-C - no cpu_atom PMU). - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 589139b07f96fa51ba8113e773723893ad3469bc Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 16:07:18 -0700 Subject: [PATCH 101/123] tune harness: the engine-shape seat fixture is built at the ffn width (n=14336, 512 rows) and tiled 96x - at n=2048 the k3 seat went the wrong way on Granite Rapids --- modules/dasLLAMA/harness/gen_tune_probe.das | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index d3ec602b1e..ebed9d13ea 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1453,13 +1453,17 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge } with_job_que() { // the lane count and the team dispatch both need the live jobque if (get_total_hw_jobs() > 1) { - let TILES = 48l // 393216 rows at n=2048: 200-660 MB per format, past any server L3 - var ktfx = KqFixture(fmt = kgfx.fmt, n = kgfx.n, d = kgfx.d * TILES, ntok = 1l, nsb = kgfx.nsb) - ktfx.xq := kgfx.xq - ktfx.xs := kgfx.xs - ktfx.xbs := kgfx.xbs + // the engine's row length matters (k3 on Granite Rapids: 256_mr16 wins at n=2048, loses at 14336), so the + // seat fixture is a short one at the ffn width, tiled: 512 rows transcode in a blink, 96 tiles make 49152 + var kefx <- build_kq_fixture(fmt, 14336l, 512l, 1l) + let TILES = 96l + var ktfx = KqFixture(fmt = kefx.fmt, n = kefx.n, d = kefx.d * TILES, ntok = 1l, nsb = kefx.nsb) + ktfx.xq := kefx.xq + ktfx.xs := kefx.xs + ktfx.xbs := kefx.xbs var tbuf : KqBuf - tile_kq_planes(kgfx, TILES, tbuf) + tile_kq_planes(kefx, TILES, tbuf) + delete kefx unsafe { repack_kq_grp_fmt(fmt, addr(tbuf.q[0]), addr(tbuf.s[0]), ktfx.n, ktfx.d, vmr[winner]) } From d98323e8d426747ae81626509524f8279b17fda6 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 16:13:55 -0700 Subject: [PATCH 102/123] kq_kernel_bench: a q8s16 arm - q8 over binary16 group scales, the wscale_f16 rail a GGUF q8_0 tensor runs; the ladder maps it to q8_0 The q8 arm times q8q8_gemv_gen over f32 group scales (9 bits/weight, 528 MB at d=32768 n=14336) against llama.cpp's q8_0 (f16 d, 8.5 bits, 499 MB) - a 6% byte handicap the engine never pays for GGUF q8_0 tensors, which run the s16 twins. q8s16 times q8q8_gemv_s16_gen / q8q8_tile_s16_gen over a 2-byte scale plane (the same 0x3C fill reads as f16 1.06). zen2 one thread: q8s16 3021 vs q8 3235 us. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../benchmarks/matmul/kq_kernel_bench.das | 63 ++++++++++++++++--- modules/dasLLAMA/harness/kernel_ladder.sh | 2 +- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 24a6215936..b3f5b9543f 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -30,7 +30,7 @@ require daslib/strings_boost [CommandLineArgs] struct BenchArgs { @clarg_short = "f" - @clarg_doc = "KqFmt names, comma-separated (q8,k4,k5,k6,q40,q51,iq4xs,k3,iq3s,iq3xxs,iq4nl,k2,iq2s,iq2xs,iq2xxs; mx4 = mxfp4), or all" + @clarg_doc = "KqFmt names, comma-separated (q8,k4,k5,k6,q40,q51,iq4xs,k3,iq3s,iq3xxs,iq4nl,k2,iq2s,iq2xs,iq2xxs; mx4 = mxfp4; q8s16 = q8 over binary16 group scales, the GGUF q8_0 tensor's path), or all" fmt : string = "all" @clarg_doc = "Reduction length - the weight row length (k in test-backend-ops terms)" n : int = 14336 @@ -62,12 +62,16 @@ struct BenchArgs { typedef GemvFn = function<(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) : void> typedef TileFn = function<(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void> typedef GemvQ8Fn = function<(var yp : float?; wp : int8 const?; sp : float const?; xqp : int8 const?; xsp : float const?; n, rb, re : int64) : void> +typedef GemvQ8S16Fn = function<(var yp : float?; wp : int8 const?; sp : uint16 const?; xqp : int8 const?; xsp : float const?; n, rb, re : int64) : void> +typedef TileQ8S16Fn = function<(var yp : float?; wg : int8 const?; sg : uint16 const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void> typedef TileQ8Fn = function<(var yp : float?; wg : int8 const?; sg : float const?; xqp : int8 const?; xsp : float const?; xbsp : int const?; n, d, g, t0 : int64) : void> typedef GemvMx4Fn = function<(var yp : float?; wn : uint8 const?; we : uint8 const?; xqp : int8 const?; xsp : float const?; n, rb, re : int64) : void> typedef LayoutFn = function<() : int> let TILE_TOKENS = 4l // tokens one kq / q51 / mx4 tile call covers (q8's tokstep companion says its own) let MX4_NAME = "mx4" // mxfp4 has no KqFmt row; it rides the q8q8 grid as a companion +let S16_NAME = "q8s16" // q8 over binary16 group scales - the wscale_f16 rail a GGUF q8_0 tensor runs (8.5 bits/weight, llama.cpp's q8_0 bytes) +var g_s16 = false // the current q8-family row is the s16 flavor def fmt_of_name(name : string) : KqFmt { for (f in type) { @@ -158,7 +162,7 @@ def plane_shape(f : KqFmt; mx4 : bool; n : int64) : PlaneShape { return PlaneShape(wq_row = n / 2l, tq_row = n / 2l, ws_row = n / 32l) } if (f == KqFmt.q8) { - return PlaneShape(wq_row = n, tq_row = n, ws_row = (n / 32l) * 4l) + return PlaneShape(wq_row = n, tq_row = n, ws_row = (n / 32l) * (g_s16 ? 2l : 4l)) } if (f == KqFmt.q51) { return PlaneShape(wq_row = (n / 32l) * 20l, tq_row = (n / 32l) * 20l, ws_row = (n / 32l) * 4l) @@ -280,6 +284,16 @@ def team_gemv_q8(fn : GemvQ8Fn; var yp : float?; wp : int8 const?; sp : float co } } +def team_gemv_q8s16(fn : GemvQ8S16Fn; var yp : float?; wp : int8 const?; sp : uint16 const?; xqp : int8 const?; xsp : float const?; n, d : int64; var laneP : int64?) { + maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { + let l0 = ref_time_ticks() + invoke(fn, yp, wp, sp, xqp, xsp, n, int64(rb), int64(re)) + unsafe { + laneP[slot] += int64(get_time_usec(l0)) + } + } +} + def team_gemv_mx4(fn : GemvMx4Fn; var yp : float?; wn, we : uint8 const?; xqp : int8 const?; xsp : float const?; n, d : int64; var laneP : int64?) { maybe_parallel_for_indexed(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(slot : int; rb : int; re : int) { let l0 = ref_time_ticks() @@ -487,7 +501,7 @@ def bench_tile_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va } //! the q8q8 grid's gemv rows (q8 or its mx4 companion) into stats + typed function lists -def q8_gemv_rows(cfg : BenchArgs; mx4 : bool; var stats : array; var fnsQ : array; var fnsM : array) { +def q8_gemv_rows(cfg : BenchArgs; mx4 : bool; var stats : array; var fnsQ : array; var fnsM : array; var fnsS : array) { if (mx4) { var vs <- mx4q8_gemv_gen_variants() for (v in vs) { @@ -497,6 +511,15 @@ def q8_gemv_rows(cfg : BenchArgs; mx4 : bool; var stats : array; var fn } } delete vs + } elif (g_s16) { + var vs <- q8q8_gemv_s16_gen_variants() + for (v in vs) { + if (perm_wanted(cfg.perm, v._0)) { + stats |> emplace(RowStat(perm = v._0)) + fnsS |> push(v._1) + } + } + delete vs } else { var vs <- q8q8_gemv_gen_variants() for (v in vs) { @@ -510,7 +533,7 @@ def q8_gemv_rows(cfg : BenchArgs; mx4 : bool; var stats : array; var fn } //! the q8q8 grid's tile rows with each row's mr and tokens-per-call; skips rows whose mr the row count does not divide -def q8_tile_rows(cfg : BenchArgs; mx4 : bool; d : int64; var stats : array; var fnsQ : array; var fnsM : array; var mrOf, stepOf : array) { +def q8_tile_rows(cfg : BenchArgs; mx4 : bool; d : int64; var stats : array; var fnsQ : array; var fnsM : array; var fnsS : array; var mrOf, stepOf : array) { var mrs <- companion_ints(q8q8_layout_gen_variants()) var steps <- companion_ints(q8q8_tokstep_gen_variants()) if (mx4) { @@ -525,6 +548,18 @@ def q8_tile_rows(cfg : BenchArgs; mx4 : bool; d : int64; var stats : array 0l && d % mr == 0l) { + stats |> emplace(RowStat(perm = v._0)) + fnsS |> push(v._1) + mrOf |> push(mr) + stepOf |> push(steps?[v._0] ?? (steps?["reference"] ?? TILE_TOKENS)) + } + } + delete vs } else { var vs <- q8q8_tile_gen_variants() for (v in vs) { @@ -547,7 +582,7 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes let n = int64(cfg.n) let d = int64(cfg.d) let ntok = int64(cfg.ntok) - let name = mx4 ? MX4_NAME : "q8" + let name = mx4 ? MX4_NAME : (g_s16 ? S16_NAME : "q8") let plane_bytes = int64(cfg.d) * (sh.wq_row + sh.ws_row) var wvs <- q8q8_family_live_variants() // the family's witness rows: the amx one takes the per-process tile permission its tiles need (SIGILL without it), as the probe does for (w in wvs) { @@ -565,11 +600,14 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes var stats : array var fnsQ : array var fnsM : array - q8_gemv_rows(cfg, mx4, stats, fnsQ, fnsM) + var fnsS : array + q8_gemv_rows(cfg, mx4, stats, fnsQ, fnsM, fnsS) if (cfg.team) { time_rows_team(stats, cfg.rounds) $(row : int; var laneP : int64?) { if (mx4) { team_gemv_mx4(fnsM[row], yp, kqp, ksp, xqp, xsp, n, d, laneP) + } elif (g_s16) { + team_gemv_q8s16(fnsS[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, d, laneP) } else { team_gemv_q8(fnsQ[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, d, laneP) } @@ -578,6 +616,8 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes time_rows(stats, cfg.rounds) $(row : int) { if (mx4) { invoke(fnsM[row], yp, kqp, ksp, xqp, xsp, n, 0l, d) + } elif (g_s16) { + invoke(fnsS[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, 0l, d) } else { invoke(fnsQ[row], yp, reinterpret(kqp), reinterpret(ksp), xqp, xsp, n, 0l, d) } @@ -589,14 +629,16 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes delete stats delete fnsQ delete fnsM + delete fnsS } if (ntok > 0l) { var stats : array var fnsQ : array var fnsM : array + var fnsS : array var mrOf : array var stepOf : array - q8_tile_rows(cfg, mx4, d, stats, fnsQ, fnsM, mrOf, stepOf) + q8_tile_rows(cfg, mx4, d, stats, fnsQ, fnsM, fnsS, mrOf, stepOf) time_rows(stats, cfg.rounds) $(row : int) { let mr = mrOf[row] for (g in range64(d / mr)) { @@ -604,6 +646,8 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes while (t0k < ntok) { if (mx4) { invoke(fnsM[row], yp, kqp + g * mr * sh.tq_row, ksp + g * mr * sh.ws_row, xqp, xsp, xbsp, n, d, g, t0k) + } elif (g_s16) { + invoke(fnsS[row], yp, reinterpret(kqp + g * mr * sh.tq_row), reinterpret(ksp + g * mr * sh.ws_row), xqp, xsp, xbsp, n, d, g, t0k) } else { invoke(fnsQ[row], yp, reinterpret(kqp + g * mr * sh.tq_row), reinterpret(ksp + g * mr * sh.ws_row), xqp, xsp, xbsp, n, d, g, t0k) } @@ -617,6 +661,7 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes delete stats delete fnsQ delete fnsM + delete fnsS delete mrOf delete stepOf } @@ -631,6 +676,7 @@ def formats_of(spec : string) : array> { fs |> emplace(("{f}", f, false)) } fs |> emplace((MX4_NAME, KqFmt.q8, true)) + fs |> emplace((S16_NAME, KqFmt.q8, false)) return <- fs } for (tok in split(spec, ",")) { @@ -640,6 +686,8 @@ def formats_of(spec : string) : array> { } if (name == MX4_NAME) { fs |> emplace((MX4_NAME, KqFmt.q8, true)) + } elif (name == S16_NAME) { + fs |> emplace((S16_NAME, KqFmt.q8, false)) } else { fs |> emplace((name, fmt_of_name(name), false)) } @@ -695,6 +743,7 @@ def run_formats(cfg : BenchArgs; fs : array>) : int { var rows = 0 let nsb = int64(cfg.n) / 256l for (fe in fs) { + g_s16 = fe._0 == S16_NAME let sh = plane_shape(fe._1, fe._2, int64(cfg.n)) var p : Planes fill_planes(p, sh, int64(cfg.n), int64(cfg.d), int64(max(cfg.ntok, 1))) diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh index d8a9bf03b2..1268239cc8 100644 --- a/modules/dasLLAMA/harness/kernel_ladder.sh +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -44,7 +44,7 @@ trap 'rm -rf "$WORK"' EXIT # KqFmt name -> ggml type name (the reference's type_a) ggml_type() { case "$1" in - q8) echo q8_0;; k4) echo q4_K;; k5) echo q5_K;; k6) echo q6_K;; q40) echo q4_0;; q51) echo q5_1;; + q8) echo q8_0;; q8s16) echo q8_0;; k4) echo q4_K;; k5) echo q5_K;; k6) echo q6_K;; q40) echo q4_0;; q51) echo q5_1;; iq4xs) echo iq4_xs;; k3) echo q3_K;; iq3s) echo iq3_s;; iq3xxs) echo iq3_xxs;; iq4nl) echo iq4_nl;; k2) echo q2_K;; iq2s) echo iq2_s;; iq2xs) echo iq2_xs;; iq2xxs) echo iq2_xxs;; mx4) echo mxfp4;; *) echo "";; From 05ec4c3eb89ba2d46c7653d91cdf1150cfa63d34 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 16:18:05 -0700 Subject: [PATCH 103/123] kernel parity plan: q8s16 like-for-like row, the prefetch null, the per-format lane sweep, Intel v6 --- plans/kernel_parity_pass.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index ef64ebb4ba..2cdbb55874 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -932,5 +932,18 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. the reference's 82 on q8 while our k4 streams at 91 on the same box: memory-level parallelism, not bandwidth. - 2026-09-01: x86 hybrid core detection (Apple-only today) is LEDGERED as followup_general.md entry 66, not an arc follow-up; the goal boxes are homogeneous (c7a EPYC 9R14, c8i Xeon 6975P-C - no cpu_atom PMU). +- 2026-09-01: the q8 ladder row was a byte handicap: the bench's q8 plane carries f32 group scales (9 bits/weight) + against llama.cpp's q8_0 f16 d (8.5) - at equal streaming rate it can never pass 0.944. The engine runs GGUF + q8_0 tensors on the wscale_f16 twins, so the bench got a q8s16 arm (2ad5b9980): zen4 16 lanes 6277 vs the + reference 5985 = 0.95 (f32 row 6673 = 0.90), both at ~79.5 GB/s. The pf=2048 software prefetch was a null + (6712 / 6722 / 6712 for pf / crown / 256_mr8). +- 2026-09-01: zen4 lane sweep at d=32768, 8 vs 16 lanes: iq4xs 3004 / 3116, iq4nl 2946 / 3049, q8s16 6120 / 6339, + k4 2891 / 3169 (streaming formats gain 3.5-9% at one lane per core - issue width at two SMT lanes, not DRAM), + k5 3997 / 3949 (flat), iq2xs 2714 / 1412 (the lattice grids halve without the SMT lanes). The Mac decode policy + transfers to SMT x86 PER FORMAT; a tuner-minted per-family decode lane cap is the proposed closing move for + zen4's iq4xs (0.92) / iq4nl (0.94); k5 sits at 0.93-0.97 across runs regardless. +- 2026-09-01: Intel v6 (rebased, seat race at the engine shape but n=2048): every row >= 0.97 except k3 0.89 - + the tiled fixture's row length flipped the k3 seat to 256_mr16 again; the seat fixture is now built at n=14336 + (29dc0b35f). Intel v7 and zen4 v7 re-mints running. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 179043eb7370e303190b2869f5dff8778612b070 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 16:24:52 -0700 Subject: [PATCH 104/123] followup_general 67: per-format decode lane cap on SMT x86 - zen4's streaming rows accepted at 0.92-0.94 --- modules/dasLLAMA/followup_general.md | 13 +++++++++++++ plans/kernel_parity_pass.md | 5 +++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 1fba341751..a15c6d8cc0 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -843,3 +843,16 @@ Windows reads `EfficiencyClass` from `GetLogicalProcessorInformationEx(RelationProcessorCore)`; the darwin branch then applies unchanged. No EC2 instance has such a part (server Xeons only); a Hetzner EX44 (i5-13500, 6P+8E) or EX101 (i9-13900) rents by the hour for the check. + +67. **Per-format decode lane cap on SMT x86.** zen4 (8 cores x 2 SMT) at the engine's decode shape + (d=32768, DRAM-streamed): the light formats gain at one lane per core - iq4xs 3116 -> 3004 us, + iq4nl 3049 -> 2946, q8s16 6339 -> 6120, k4 3169 -> 2891 - while the lattice grids halve without + the SMT lanes (iq2xs 1412 -> 2714) and k5 does not move. Two siblings split one core's issue + width on a kernel that is not ALU-bound; the reference streams the same bytes at 84-91 GB/s + where ours sit at 79-82 with every lane busy. The darwin policy (`dasllama_jobque_threads_cap`: + every core for prefill, perf cores for decode) is the right shape but one global cap; here it + has to be per family: the tuner races the gemv seat at physical-core lanes beside the all-lane + race it already runs, stores `decode_lanes` per family in the sidecar, and `matmul_chunks_gemv` + applies the format's cap (its callers know the tensor's KqFmt). Ruled 2026-09-01: zen4's iq4xs + 0.92 / iq4nl 0.94 / k5 0.93-0.97 are accepted as they stand; pinned affinity is normally the + faster arrangement and this may be the one case it is not - decide with a second SMT box in hand. diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 2cdbb55874..f8f4af3290 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -940,8 +940,9 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. - 2026-09-01: zen4 lane sweep at d=32768, 8 vs 16 lanes: iq4xs 3004 / 3116, iq4nl 2946 / 3049, q8s16 6120 / 6339, k4 2891 / 3169 (streaming formats gain 3.5-9% at one lane per core - issue width at two SMT lanes, not DRAM), k5 3997 / 3949 (flat), iq2xs 2714 / 1412 (the lattice grids halve without the SMT lanes). The Mac decode policy - transfers to SMT x86 PER FORMAT; a tuner-minted per-family decode lane cap is the proposed closing move for - zen4's iq4xs (0.92) / iq4nl (0.94); k5 sits at 0.93-0.97 across runs regardless. + transfers to SMT x86 PER FORMAT. RULED (Boris, 2026-09-01): ledgered as followup_general.md entry 67, not built + this arc - zen4's iq4xs 0.92 / iq4nl 0.94 / k5 0.93-0.97 are accepted; pinned affinity is normally faster and + this may be the one case it is not. - 2026-09-01: Intel v6 (rebased, seat race at the engine shape but n=2048): every row >= 0.97 except k3 0.89 - the tiled fixture's row length flipped the k3 seat to 256_mr16 again; the seat fixture is now built at n=14336 (29dc0b35f). Intel v7 and zen4 v7 re-mints running. From 317a9a11578049771777c2ae0b40b4ff83eac8cc Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 16:37:32 -0700 Subject: [PATCH 105/123] tune harness: the engine-shape seat fixture beats any L3 (320 tiles, 600 MB - 1.9 GB) and the seat takes the median of seven rounds On Granite Rapids a 300 MB k3 plane came back L3-resident in some rounds (1478 us = 137 GB/s against 3000 from DRAM; the socket lends its 504 MB L3 to the slice) and best-of crowned the cached-mode winner - 256_mr16, which loses 10% to the 512 body at DRAM rates (the ladder's 0.87). The fixture now spans 163840 rows at n=14336 and the seat compares medians. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/harness/gen_tune_probe.das | 22 ++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index ebed9d13ea..adaa9b5440 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -1454,9 +1454,11 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge with_job_que() { // the lane count and the team dispatch both need the live jobque if (get_total_hw_jobs() > 1) { // the engine's row length matters (k3 on Granite Rapids: 256_mr16 wins at n=2048, loses at 14336), so the - // seat fixture is a short one at the ffn width, tiled: 512 rows transcode in a blink, 96 tiles make 49152 + // seat fixture is a short one at the ffn width, tiled: 512 rows transcode in a blink; 320 tiles make 163840 + // rows - 600 MB for the 2-bit grids, 1.9 GB for k6 - past the 504 MB L3 a Granite Rapids socket lends a + // slice (a 300 MB plane came back L3-resident in some rounds and the seat went to the cached-mode winner) var kefx <- build_kq_fixture(fmt, 14336l, 512l, 1l) - let TILES = 96l + let TILES = 320l var ktfx = KqFixture(fmt = kefx.fmt, n = kefx.n, d = kefx.d * TILES, ntok = 1l, nsb = kefx.nsb) ktfx.xq := kefx.xq ktfx.xs := kefx.xs @@ -1472,7 +1474,9 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge for (i in range(nv)) { tbest[i] = 1.0e30lf } - for (r in range(5)) { + var tsamples : array> + tsamples |> resize(nv) + for (r in range(7)) { // the seat takes the MEDIAN: a round that finds the plane in L3 must not crown it for (vi in range(nv)) { continue if (vmr[vi] != vmr[winner] || best[vi] >= 1.0e29lf) for (_w in range(r == 0 ? 3 : 0)) { @@ -1480,12 +1484,16 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge } let t0 = ref_time_ticks() run_kq_gemv_team(gtab[tvs[vi]._0], ktfx, tbuf.q, tbuf.s, vmr[vi], kyt) - let us = double(get_time_usec(t0)) - if (us < tbest[vi]) { - tbest[vi] = us - } + tsamples[vi] |> push(double(get_time_usec(t0))) + } + } + for (vi in range(nv)) { + if (length(tsamples[vi]) > 0) { + sort(tsamples[vi]) + tbest[vi] = tsamples[vi][length(tsamples[vi]) / 2] } } + delete tsamples for (vi in range(nv)) { if (vmr[vi] == vmr[winner] && best[vi] < 1.0e29lf) { tune_detail("k{fmt} {tvs[vi]._0}: decode at the engine shape {tbest[vi]} us ({get_total_hw_jobs()} lanes, d={ktfx.d} n={ktfx.n})\n") From 33ed27a930c4d05e6af3a1ae212360feb9e0f96b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 16:59:54 -0700 Subject: [PATCH 106/123] tune profile x86-amx: re-minted with the L3-proof engine-shape seat race - every ladder row at or past 0.95 of llama.cpp 56 entries, provenance ok/ok. 16 lanes: q8 0.96, q8s16 1.02, k6 1.06, k3 0.95, the grids 1.17-2.35; one thread every row >= 1.15; tiles clean. The Intel half of the goal. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../defaults/x86-amx.tune-defaults.json | 32 ++++++++----------- plans/kernel_parity_pass.md | 24 ++++++++++++++ 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json index 076428a928..e3bdde0391 100644 --- a/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-amx.tune-defaults.json @@ -1,10 +1,10 @@ { "kernels" : { "iq4xsq8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "q40q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", "add_inplace" : "vec16", "iq2xsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "cvt_f32_to_f16" : "vec16", - "iq2sq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "rope_scaled_neox_tab" : "vec16", "q51q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "softmax" : "vec8_u2", @@ -16,34 +16,30 @@ "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "iq4nlq8_tile_gen" : "dot_vpdpbusd_width512_mr16", "quantize_q8kv_row" : "plain", - "axpy_f16" : "vec16_u2", - "q40q8_tile_gen" : "dot_vpdpbusd_width256_mr16", + "axpy_f16" : "vec16", + "q40q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "axpy_tq4kv" : "vec8_u2", "cvt_tq4kv_to_f32" : "vec8_u2", - "axpy" : "vec16_u2", + "axpy" : "vec16", "dot_q8q8kv" : "vec16_u2", "dot_mx4q8" : "u2", "softmax_sink" : "vec8_u2", - "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "dot_q8q8_laneq4x4" : "", "dot_bf16" : "vec8_u2", - "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "add_scale_inplace" : "vec8_u2", "cvt_q8kv_to_f32" : "vec16_u2", "axpy_q8kv" : "vec16_u2", "dot_q8q8_f16s" : "vec16", - "iq2xxsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", - "k5q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "k6q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", "k4q8_tile_gen" : "dot_vpdpbusd_width256_mr16", "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "k6q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", - "k3q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "k3q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "k3q8_tile_gen" : "dot_vpdpbusd_width256_mr16", "iq4nlq8_gemv_gen" : "dot_vpdpbusd_width256_mr16", - "k2q8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "k2q8_gemv_gen" : "dot_vpdpbusd_width256_mr16", + "k2q8_tile_gen" : "dot_vpdpbusd_width256_mr16", "quantize_q8_0_into_ptr" : "plain", "gemm_f32_uk_4x16" : "u2", "dot_q51e" : "vec16", @@ -63,18 +59,18 @@ }, "provenance" : { "validation" : "ok", - "noise_probes" : "start cv 0.41%; mid1 cv 0.04%; mid2 cv 0.08%; end cv 0.20%", + "noise_probes" : "start cv 0.08%; mid1 cv 0.04%; mid2 cv 0.12%; end cv 0.05%", "platform" : "linux", - "noise_floor_cv_pct" : "0.41", + "noise_floor_cv_pct" : "0.12", "features" : "avx2;f16c;fma;sse4.2;avxvnni;avx512f;avx512bw;avx512vl;avx512vnni;avx512vbmi;amx-tile;amx-int8", "class" : "x86-amx", - "written" : "2026-09-01T21:21:48.426Z", - "validation_demoted" : "1", + "written" : "2026-09-01T23:48:21.884Z", + "validation_demoted" : "0", "mode" : "normal", "dasllama_version" : "15", "origin" : "profile", "noise" : "ok", "arch" : "x86_64", - "validation_max_drift_pct" : "5.13" + "validation_max_drift_pct" : "6.95" } } \ No newline at end of file diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index f8f4af3290..db3ce70439 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -518,6 +518,30 @@ Reading: every one-thread row on both boxes is past the reference (zen4 >= 1.10, Under 0.95 at 16 lanes: zen4 q8 0.89, iq4xs 0.92, q51 0.94 (the DRAM-bound rows; iq4nl exactly 0.95) and Intel k6 0.68 (the normal-mode stall gap). Both minted profiles committed; the gemv seats took grid_vbmi for all five grid families on both boxes. +### Intel v8 - THE GOAL TABLE (2026-09-01, c8i.4xlarge Xeon 6975P-C, x86-amx re-minted with the L3-proof engine-shape seat race, a3a1a08ab; ratio = reference / ours) + +| fmt | gemv 1T | tile 1T | gemv 16L | +|---|---|---|---| +| q8 | 1.16 | 3.76 | 0.96 | +| q8s16 | 1.19 | 3.78 | 1.02 | +| k4 | 1.48 | 3.31 | 1.53 | +| k5 | 1.61 | 6.71 | 1.51 | +| k6 | 1.18 | 3.78 | 1.06 | +| q40 | 2.05 | 3.64 | 2.78 | +| q51 | 1.94 | 6.35 | 1.87 | +| iq4xs | 2.10 | 4.11 | 2.35 | +| k3 | 1.60 | 3.24 | 0.95 | +| iq3s | 2.13 | 14.08 | 1.86 | +| iq3xxs | 1.99 | 10.32 | 2.18 | +| iq4nl | 2.37 | 2.68 | 3.30 | +| k2 | 1.61 | 2.13 | 1.44 | +| iq2s | 1.15 | 6.83 | 1.17 | +| iq2xs | 1.55 | 8.00 | 1.64 | +| iq2xxs | 1.59 | 7.27 | 1.73 | +| mx4 | 2.44 | 4.78 | 2.79 | + +Every row at or past 0.95: the goal holds on Intel. q8s16 is the like-for-like q8_0 row (f16 scales); the f32-scale q8 row is informational (byte-capped at 0.944 in theory, 0.96 here). + ## 3. Research memos (read before touching the kernels) - `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, From 12e0dc64d689f4cdacf0ab140306373ff60461ec Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 17:04:32 -0700 Subject: [PATCH 107/123] tune profile x86-vnni512: re-minted with the L3-proof engine-shape seat race; the zen4 goal table 49 entries, provenance ok/ok; the five grid families' tile crowns are the vbmi rows (a tile tie with the 512 body they share), so their gemv companions carry the lattice. 16 lanes: every grid 1.19-1.51, k3/k6/q51/mx4/k2 >= 0.99, q8s16 0.96; the accepted streaming set (iq4xs, k5, iq4nl, q40) 0.91-0.99 across runs and the f32-scale q8 at its 0.944 byte cap. One thread every row >= 1.07. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../defaults/x86-vnni512.tune-defaults.json | 27 +++++++------------ plans/kernel_parity_pass.md | 27 +++++++++++++++++++ 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json index bcb0cbe008..3584d8353e 100644 --- a/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/x86-vnni512.tune-defaults.json @@ -1,15 +1,13 @@ { "kernels" : { "add_inplace" : "vec8_u2", - "iq2xsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "cvt_f32_to_f16" : "plain", - "iq2sq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "rope_scaled_neox_tab" : "plain", "q51q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "softmax" : "vec8_u2", "mul_inplace" : "vec8_u2", "quantize_q8_0_bs_into_ptr" : "u2", - "iq2xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq2xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "dot_q8kv" : "vec8_u2", "dot_q8q8" : "u2", "iq4xsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", @@ -23,34 +21,29 @@ "dot_q8q8kv" : "u2", "dot_mx4q8" : "u2", "softmax_sink" : "vec8_u2", - "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq2xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "dot_q8q8_laneq4x4" : "", "dot_bf16" : "vec8_u2", - "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq2sq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "add_scale_inplace" : "vec8_u2", "cvt_q8kv_to_f32" : "vec8_u2", "axpy_q8kv" : "plain", "dot_q8q8_f16s" : "vec16", - "iq2xxsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "q8q8_tile_gen" : "dot_vpdpbusd_width512_mr16_kstep2_gkstep2_bias128", - "k3q8_gemv_gen" : "dot_vpdpbusd_width256_mr8", + "quantize_q8_0_into_ptr" : "plain", "k4q8_tile_gen" : "dot_vpdpbusd_width256_mr16", "k5q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "k6q8_tile_gen" : "dot_vpdpbusd_width512_mr16", "k3q8_tile_gen" : "dot_maddubs_width256_mr8", "k2q8_tile_gen" : "dot_vpdpbusd_width256_mr16", - "k2q8_gemv_gen" : "dot_vpdpbusd_width512_mr16", - "quantize_q8_0_into_ptr" : "plain", "gemm_f32_uk_4x16" : "u2", "dot_q51e" : "vec16", "dot_f16" : "vec16", "cvt_f16_to_f32" : "vec8_u2", "dot" : "vec16", - "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16", - "iq3xxsq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", - "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16", + "iq3xxsq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", + "iq3sq8_tile_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "dot_q8tq4kv" : "vec16", - "iq3sq8_gemv_gen" : "dot_vpdpbusd_width512_mr16_grid_vbmi", "scale_inplace" : "vec8_u2", "dot_q4" : "vec8_u2", "quantize_tq4kv_row" : "plain", @@ -59,18 +52,18 @@ }, "provenance" : { "validation" : "ok", - "noise_probes" : "start cv 0.10%; mid1 cv 0.05%; mid2 cv 0.36%; end cv 0.05%", + "noise_probes" : "start cv 0.07%; mid1 cv 0.24%; mid2 cv 0.05%; end cv 0.08%", "platform" : "linux", - "noise_floor_cv_pct" : "0.36", + "noise_floor_cv_pct" : "0.24", "features" : "avx2;f16c;fma;sse4.2;avx512f;avx512bw;avx512vl;avx512vnni;avx512vbmi", "class" : "x86-vnni512", - "written" : "2026-09-01T21:22:56.997Z", + "written" : "2026-09-01T23:49:31.046Z", "validation_demoted" : "1", "mode" : "normal", "dasllama_version" : "15", "origin" : "profile", "noise" : "ok", "arch" : "x86_64", - "validation_max_drift_pct" : "2.85" + "validation_max_drift_pct" : "23.00" } } \ No newline at end of file diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index db3ce70439..9c9bc2e55e 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -542,6 +542,33 @@ Both minted profiles committed; the gemv seats took grid_vbmi for all five grid Every row at or past 0.95: the goal holds on Intel. q8s16 is the like-for-like q8_0 row (f16 scales); the f32-scale q8 row is informational (byte-capped at 0.944 in theory, 0.96 here). +### zen4 v8 - THE GOAL TABLE (2026-09-01, c7a.4xlarge EPYC 9R14, x86-vnni512 re-minted with the L3-proof engine-shape seat race, a3a1a08ab; ratio = reference / ours) + +| fmt | gemv 1T | tile 1T | gemv 16L | +|---|---|---|---| +| q8 | 1.99 | 3.01 | 0.90 (ladder pass read 0.68; re-measured 6677-6703 vs 6059) | +| q8s16 | 1.92 | 2.99 | 0.96 | +| k4 | 2.29 | 2.49 | 0.98 | +| k5 | 1.33 | 6.30 | 0.98 | +| k6 | 1.07 | 3.47 | 0.99 | +| q40 | 4.48 | 3.42 | 0.91-0.97 (ladder 0.89; re-measured 2944-3143 vs 2851) | +| q51 | 2.83 | 6.02 | 1.01 | +| iq4xs | 2.17 | 4.05 | 0.91 | +| k3 | 1.79 | 2.69 | 1.01 | +| iq3s | 2.24 | 11.91 | 1.30 | +| iq3xxs | 2.22 | 10.23 | 1.19 | +| iq4nl | 2.86 | 2.20 | 0.99 | +| k2 | 1.74 | 1.69 | 1.10 | +| iq2s | 1.33 | 6.71 | 1.28 | +| iq2xs | 1.44 | 6.52 | 1.44 | +| iq2xxs | 1.79 | 7.66 | 1.51 | +| mx4 | 2.60 | 4.60 | 1.02 | + +One thread every row >= 1.07, tiles clean, every grid row 1.19-1.51 at 16 lanes. Under 0.95 at 16 lanes: the accepted streaming +set (ruled, followup_general 67) - iq4xs 0.91-0.94, k5 0.93-0.98, iq4nl 0.94-0.99, q40 0.91-0.97 - and the f32-scale q8 row, +byte-capped at 0.944 (q8s16, the GGUF q8_0 path, reads 0.96). The grid families' tile crowns are the vbmi rows (a tile tie - +the lattice row's tile IS the 512 body - so the gemv companion carries the lattice without a separate seat). + ## 3. Research memos (read before touching the kernels) - `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, From 8c07d77dff7b402e0fef1a50b760eba7e197e259 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 17:16:53 -0700 Subject: [PATCH 108/123] dasLLVM harvest: the x64 tier-gate inventory (ARCHITECTURE.md#x64-tier-gates, cited from init_jit_target_flags), two REVIEW rules, the companion-seat rename The GEP wrapper's inbounds rule and the three-part tier-feature rule (cpuid table + TUNE_KNOWN_FEATURES + gate) land in REVIEW.md; the nine g_target_x64_* gates and how each is decided land as an anchored section; csuffix -> companionOwnSuffix, own -> stampedOwn. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLVM/ARCHITECTURE.md | 19 +++++++++++++++++++ modules/dasLLVM/REVIEW.md | 10 ++++++++++ modules/dasLLVM/daslib/llvm_boost.das | 3 --- modules/dasLLVM/daslib/llvm_jit_common.das | 4 ++-- modules/dasLLVM/daslib/llvm_tune.das | 22 +++++++++++----------- 5 files changed, 42 insertions(+), 16 deletions(-) diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md index 59a878dba9..41552bd0c0 100644 --- a/modules/dasLLVM/ARCHITECTURE.md +++ b/modules/dasLLVM/ARCHITECTURE.md @@ -161,3 +161,22 @@ because the mints that matter most produce no successful read - the first mint h and a re-mint replaces one the gate rejected. An absent file registers as size -1 and hash 0, which the next run's re-validation sees change. Registering is a no-op outside compilation, so the manifest's runtime readers reach the same call unconditionally. + +## 6. The x64 kernel-matrix tier gates {#x64-tier-gates} + +An x64 host target publishes nine boolean gates (`g_target_x64_*`), one per instruction tier the +kernel matrix and the tune grids select on: `avx2`, `f16c`, `vnni256` (256-bit VPDPBUSD by either +VEX AVX-VNNI or EVEX AVX512-VNNI+VL), `avx512bw` (zmm byte ops - BW, not merely F), `avx512vnni` +(zmm VPDPBUSD; implies bw, the sign trick around it is BW), `avx512vbmi` (VPERMI2B / VPERMB / +VPMULTISHIFTQB - the grid formats' symbol lattice), `vnniint8` (VEX VPDPBSSD, native s8 x s8), +and `amx` (both amx-tile and amx-int8; the per-process XTILEDATA grant is a separate runtime step +the family's own witness performs). `init_jit_target_flags` decides each from cpuid truth OR'd with +the `DAS_JIT_X64_FORCE_FEATURES` emission-only override, and forced-only on a cross triple; a +generic target (`host_features = false`) leaves every gate off. + +The cpuid truth is `das_cpu_supports` (`src/builtin/module_builtin_runtime.cpp`), a hand-kept table +keyed by the LLVM target-feature spelling - so the force env and `llc -mattr` take the same names. +A tier flag therefore has three parts that must land together: its cpuid line there, its name in +`TUNE_KNOWN_FEATURES` (`daslib/llvm_tune.das`, the profile fingerprint the `requires=` gates are +checked against), and the `g_target_x64_*` gate. A name missing from the table answers false on +every box - the perm that requires it declines everywhere and no error names the cause. diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 1607024a98..a0e32420a4 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -71,3 +71,13 @@ recorded path is read by other people and must not name the user who minted it. A host path a diff passes to a filesystem call stays raw: no filesystem call resolves `~`. `tests/llvm_tune_manifest.das` here asserts a minted sidecar carries no home directory. +- **Never set the in-bounds flag on a GEP after building it - `LLVMSetIsInBounds` casts its + argument to `GetElementPtrInst`; pass `inbounds` to `LLVMBuildGEP2` in `daslib/llvm_boost.das`, + or call `LLVMBuildInBoundsGEP2`, instead.** A GEP over a global with a constant index folds + into a `ConstantExpr`, so that cast writes the flag through a wrong type into LLVM's constant + object. +- **A new x64 tier feature lands in three places in one diff: its cpuid line in `das_cpu_supports` + (`src/builtin/module_builtin_runtime.cpp`), its name in `TUNE_KNOWN_FEATURES` + (`daslib/llvm_tune.das`), and its `g_target_x64_*` gate in `daslib/llvm_jit_common.das` + (`ARCHITECTURE.md#x64-tier-gates`).** A name the cpuid table does not know answers false on + every box, so every perm that requires it silently declines to its fallback. diff --git a/modules/dasLLVM/daslib/llvm_boost.das b/modules/dasLLVM/daslib/llvm_boost.das index ba0df73292..7de9ff92fd 100644 --- a/modules/dasLLVM/daslib/llvm_boost.das +++ b/modules/dasLLVM/daslib/llvm_boost.das @@ -307,9 +307,6 @@ def LLVMAddIncoming(phi : LLVMOpaqueValue?; var phi_vals : array starts_with("wasm") diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index 6188de5852..e44f23d2e0 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -1653,14 +1653,14 @@ class private TuneAnnotation : AstFunctionAnnotation { suffix = "" } } - var csuffix : array // a companion's OWN entry (the harness's gemv seat) when it names an eligible perm of this grid; else it follows the main - csuffix |> resize(length(comps)) + var companionOwnSuffix : array // a companion's OWN entry (the harness's gemv seat) when it names an eligible perm of this grid; else it follows the main + companionOwnSuffix |> resize(length(comps)) if (manifestFound) { for (ci, c in count(), comps) { let own = manifest?["{c.fnname}"] ?? "" for (p in perms) { if (p.suffix == own && perm_requires_ok(p)) { - csuffix[ci] = own + companionOwnSuffix[ci] = own } } } @@ -1698,19 +1698,19 @@ class private TuneAnnotation : AstFunctionAnnotation { if (p.suffix == suffix) { stamp_llvm_code(func, genKey, p, fromManifest ? mpath : "") for (ci, c in count(), comps) { - var own = false - if (!empty(csuffix[ci]) && csuffix[ci] != suffix) { + var stampedOwn = false + if (!empty(companionOwnSuffix[ci]) && companionOwnSuffix[ci] != suffix) { for (q in perms) { - if (q.suffix == csuffix[ci]) { + if (q.suffix == companionOwnSuffix[ci]) { stamp_llvm_code(c.fn, c.genkey, q, mpath) - own = true + stampedOwn = true } } - if (own && tune_verbosity() == "verbose") { - print("llvm_tune: {c.fnname} <- {csuffix[ci]} (its own entry, {mpath})\n") + if (stampedOwn && tune_verbosity() == "verbose") { + print("llvm_tune: {c.fnname} <- {companionOwnSuffix[ci]} (its own entry, {mpath})\n") } } - if (!own) { + if (!stampedOwn) { stamp_llvm_code(c.fn, c.genkey, p, fromManifest ? mpath : "") } } @@ -1735,7 +1735,7 @@ class private TuneAnnotation : AstFunctionAnnotation { } } } - delete csuffix + delete companionOwnSuffix delete perms delete comps return ok From 9d55c0c6a6621bdc234125a84f980caae5a5724b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 17:38:30 -0700 Subject: [PATCH 109/123] dasLLAMA harvest: three anchored mechanisms (the sub-block k3/k6 planes, the grid decode forms, the VBMI lattice) in ARCHITECTURE_CPU_KERNELS.md, two in ARCHITECTURE_MEASUREMENT.md (the gemv seat, the bench fixture), cited from every function that embodies them 189 added comments: 16 facts land as anchored sections with 11 [arch] citations across the emitter, the reference decodes, the repack, the layout gather, the harness and the bench; one REVIEW rule (reserve before a dimension-scaled resize); the undeclared DASLLAMA_GRID_ROWS_X86 knob is gone (the tune rows are the lab); frame_align64 is gone (an alignment experiment the measurements never confirmed); renames: TileEmit.gridPanel / gridGlobal, the bench's gemvKq / tileKq and gemvW_row / tileW_row; the by-value-table frame rule was rejected (its premise, the frame-size crash, was disproved by the GEP wrapper finding). Lint: zero warnings on the changed set, LINT026/027 clean over both modules. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/ARCHITECTURE.md | 7 +- modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md | 45 +++++ modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md | 29 +++ modules/dasLLAMA/REVIEW.md | 5 + .../benchmarks/matmul/kq_kernel_bench.das | 74 +++---- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 181 +++++++----------- modules/dasLLAMA/dasllama/dasllama_layout.das | 5 +- modules/dasLLAMA/dasllama/dasllama_math.das | 4 +- .../dasllama/dasllama_math_default.das | 4 +- .../dasLLAMA/dasllama/dasllama_math_gen.das | 41 ++-- modules/dasLLAMA/dasllama/dasllama_repack.das | 11 +- modules/dasLLAMA/harness/gen_tune_probe.das | 24 +-- modules/dasLLAMA/harness/kernel_ladder.sh | 4 +- 13 files changed, 239 insertions(+), 195 deletions(-) create mode 100644 modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index 2bbde3e57c..17a57f4708 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -55,8 +55,11 @@ re-transcoding `$LCPP/src/unicode-data.cpp`). MoE region split. - `ARCHITECTURE_MEDIA.md` - sec.2.13-2.16: the padded tower GEMM widths, the family GPU hooks, the tower weight lane, and the plain-Model ASR decoders. -- `ARCHITECTURE_MEASUREMENT.md` - sec.2.5, 2.10, 2.20: the benchmark rig, the tune gate, and the - sanctioned instrumentation rails. +- `ARCHITECTURE_MEASUREMENT.md` - sec.2.5, 2.10, 2.20, 2.21, 2.26-2.27: the benchmark rig, the tune + gate, the sanctioned instrumentation rails, kernel-race fidelity, the gemv's own tune seat, and + the CPU kernel bench's fixture conditions. +- `ARCHITECTURE_CPU_KERNELS.md` - sec.2.22-2.24: the sub-block-packed k3/k6 planes, the grid + formats' panel and row-group decodes, and the VBMI symbol lattice. ## 3. Inherited invariants diff --git a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md new file mode 100644 index 0000000000..9a32bf3b44 --- /dev/null +++ b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md @@ -0,0 +1,45 @@ +# dasLLAMA architecture - the CPU kernel planes and decodes + +Companion of `ARCHITECTURE.md` (contract: `../../ARCHITECTURE_COMMON.md`). Section 2 here continues +the mechanism numbering; each section is cited by the code that embodies it. + +## 2. Mechanisms + +### 2.22 The k3 and k6 planes are packed per sub-block {#kq-subblock-planes} + +A k6 grp plane's qh columns `2blk` and `2blk + 1` carry one sub-block's four `j` sites, each as +a 2-bit field at bit `2j`; the disk byte `h*32 + half*16 + j*4 + t` feeds sub-blocks `4h..` at bit +`2b`. k3 packs the same way - qs columns `2blk + half` carry the sub-block's four `j` sites at `2j`, +and the hmask column `blk` carries its eight sites at bit `s` (lo at `j`, hi at `4 + j`). One +sub-block's decode then costs two loads for k6 and three for k3, and nothing loaded lives past it; +the row-interleaved disk order cost one load per `j`. The layout is CPU-flavor: the `.dlim` a box +bakes is for the hardware that runs it, so a CPU plane owes nothing to the GPU tiers' shapes. +`IMAGE_VERSION` 28 is this layout. + +### 2.23 A grid format's CPU gemv decodes as a panel or as row groups {#grid-decode-forms} + +Five formats (iq3s, iq3xxs, iq2s, iq2xs, iq2xxs) have two gemv decode forms. The PANEL form gathers +a superblock into an alloca panel first and reads packed positions through one dword load per +4-byte column, the four positions of a column sharing the load. The ROW-GROUP form composes a +weight-width vector straight from the grid words - width/64 rows x 8 weights, one u64 grid entry +per iq2 row - and reverts to byte loads, because the column dword read only pays inside the panel. +The sdot lattice always takes row groups; on x86 the panel's latency chain does not scale with the +core, so `x86-vnni512` takes row groups for iq2xxs and `x86-amx` for iq2xxs and iq3xxs, everything +else the panel. A VBMI seat (sec.2.24) takes row groups unconditionally. + +### 2.24 The VBMI symbol lattice {#vbmi-lattice} + +On a zmm VBMI target a grid block decodes as row groups through a symbol lattice. Every grid byte +comes from a tiny alphabet (three symbols for the iq2 family, eight for iq3), so the grid is baked +as two compact code planes - entry `e`'s low and high half, four 2-bit symbols each for iq2, two +3-bit for iq3 - plus the alphabet as a per-lane `vpshufb` table and `ksigns_iq2xs` whole (128 bytes: +exactly the two registers one `VPERMI2B` indexes). Per format the block's index bytes gather into one +64-lane vector per column (lane `r*4 + position`) and look up in the code planes - `VPERMI2B` per 128 +entries, index bit 7 blends the pairs, the 9th and 10th index bits arrive as lane masks (iq3s and iq2s +from the row's qh byte, iq2xs from bit 0 of its u16 word's high byte). The row's sign bytes land in +the same lane layout: the plane's own column for iq3s and iq2s, one ksigns `VPERMI2B` over the 7-bit +codes for the rest. Per row group and weight octet a constant two-source shuffle places each row's +code bytes in its qword, `VPMULTISHIFTQB` spreads the symbols into bytes, one `vpshufb` maps them to +magnitudes, and the signs ride the activation copy as a mask `(x ^ m) - m`. The lattice row shares +its tile body and planes with the 512/mr16 row, so only the gemv differs - what the gemv's own seat +(`ARCHITECTURE_MEASUREMENT.md` sec.2.26) races. diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index f46cc5233c..9ac70a97f2 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -160,3 +160,32 @@ no model, no tuner, every arm gated against a CPU plane-dequant oracle before it chain every dispatch through ONE shared output buffer on purpose - the serialized regime is the instrument's probe shape, imitating the reference tool it is compared against - and its numbers reach the engine only through a human porting decision, never a minted crown. + +### 2.26 The gemv takes its own tune seat {#gemv-seat} + +A kq family's manifest entry is its tile-best row, and the gemv gets a SECOND entry when a different +row serves the streamed decode better. Only same-mr rows can differ, because the layout companion +pins the plane's interleave; of those the two best by tile time race, the winner takes the gemv only +by the margin over the tile winner's own gemv, and the incumbent keeps a tie. Every family's perm grid +therefore carries a 256-wide `mr = 16` alternate beside its 512-wide tile crown. The seat is decided +at the engine's decode shape - a DRAM-bound plane streamed by every lane through the engine's own +splitter - because the engine's row length moves the answer (k3 on Granite Rapids: the 256 seat wins +at n=2048 and loses at 14336). The seat fixture is a 512-row build at the ffn width tiled 320 times, +past the largest L3 a socket lends a slice of, and the seat takes the MEDIAN of seven rounds: a round +that finds the plane in L3 must not crown it. In normal mode `llvm_tune` stamps a companion from its +own manifest entry when one exists and is a perm this box can run, else from the tile's. + +### 2.27 The CPU kernel bench's fixture conditions {#cpu-kernel-bench-fixture} + +`benchmarks/matmul/kq_kernel_bench.das` times raw kernels on synthetic planes, and three fixture +properties decide whether its numbers mean anything. Every plane of one format lives in ONE arena at +fixed offsets, staggered so no two starts share their low 12 address bits: the heap places separate +arrays at run-dependent relative addresses, and planes that alias in the L1/L2 set logic make a run's +time depend on where the heap put them. Scale planes are filled with a byte that is a normal number in +every scale form, never random bytes, because denormal math runs orders of magnitude slower. Each row +is warmed before it is timed - three unmeasured rounds solo, six dispatches per row on the team arm - +because a core ramps over several rounds and one warm call is not enough. The q8 row exists in two +flavors: f32 group scales (the engine's own quantization) and `q8s16` over binary16 scales - the +wscale_f16 rail a GGUF q8_0 tensor runs, and the like-for-like row against the reference's q8_0. +Provenance for every figure in this section: `benchmarks/matmul/kq_kernel_bench.das` under +`DAS_TUNE_MODE=tune`, one thread, its default `--fmt` / `-n` / `-d` shape. diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 415578528f..9d0bec2b5e 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -249,3 +249,8 @@ the tuning the profile was meant to save. root) - is a `def` returning it, never a module global with a declaration initializer (`let` or `var`).** A team lane never runs global initializers, so the global reads zero there while every single-threaded run reads the right value. +- **A `resize` of a buffer whose element count scales with a model dimension in `dasllama/` is + preceded by a `reserve` of the same count** (`reserve_resize` / `grow_resize` / + `ensure_length` in `dasllama_common.das`, or the pair spelled out). A bare grow past 64 MB trips + the heap's unreserved-size cap and panics the load; `dasllama_repack.das`'s plane-size scratch + copies were the last bare ones. diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index b3f5b9543f..0e179fdf43 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -70,7 +70,7 @@ typedef LayoutFn = function<() : int> let TILE_TOKENS = 4l // tokens one kq / q51 / mx4 tile call covers (q8's tokstep companion says its own) let MX4_NAME = "mx4" // mxfp4 has no KqFmt row; it rides the q8q8 grid as a companion -let S16_NAME = "q8s16" // q8 over binary16 group scales - the wscale_f16 rail a GGUF q8_0 tensor runs (8.5 bits/weight, llama.cpp's q8_0 bytes) +let S16_NAME = "q8s16" // q8 over binary16 group scales - the wscale_f16 rail a GGUF q8_0 tensor runs (8.5 bits/weight, the reference build's q8_0 bytes) var g_s16 = false // the current q8-family row is the s16 flavor def fmt_of_name(name : string) : KqFmt { @@ -152,37 +152,36 @@ def tile_kq_bytes_per_sb(f : KqFmt) : int64 => kq_reads_packed_planes(int64(kq_s //! plane bytes per row: kq formats per 256-superblock, q51 per 32-block (20 + 4), q8 n int8 + n/32 f32, mx4 n/2 nibbles + n/32 e8m0 struct PlaneShape { - wq_row : int64 // the gemv's weight plane bytes per row - tq_row : int64 // the tile's weight plane bytes per row + gemvW_row : int64 // the gemv's weight plane bytes per row + tileW_row : int64 // the tile's weight plane bytes per row ws_row : int64 // scale plane bytes per row } def plane_shape(f : KqFmt; mx4 : bool; n : int64) : PlaneShape { if (mx4) { - return PlaneShape(wq_row = n / 2l, tq_row = n / 2l, ws_row = n / 32l) + return PlaneShape(gemvW_row = n / 2l, tileW_row = n / 2l, ws_row = n / 32l) } if (f == KqFmt.q8) { - return PlaneShape(wq_row = n, tq_row = n, ws_row = (n / 32l) * (g_s16 ? 2l : 4l)) + return PlaneShape(gemvW_row = n, tileW_row = n, ws_row = (n / 32l) * (g_s16 ? 2l : 4l)) } if (f == KqFmt.q51) { - return PlaneShape(wq_row = (n / 32l) * 20l, tq_row = (n / 32l) * 20l, ws_row = (n / 32l) * 4l) + return PlaneShape(gemvW_row = (n / 32l) * 20l, tileW_row = (n / 32l) * 20l, ws_row = (n / 32l) * 4l) } let nsb = n / 256l - return PlaneShape(wq_row = nsb * kq_qsb(f), tq_row = nsb * tile_kq_bytes_per_sb(f), ws_row = nsb * kq_ssb(f)) + return PlaneShape(gemvW_row = nsb * kq_qsb(f), tileW_row = nsb * tile_kq_bytes_per_sb(f), ws_row = nsb * kq_ssb(f)) } var BASE_ALIGN = 64l // the arena's start alignment - --base-align; the planes' phase against 4 KB pages follows it var BASE_OFFSET = 0l // --base-offset, added past the aligned start -let ALIGN = 64l // the image's planes are page-aligned (ARCHITECTURE_IMAGE.md, image-page-alignment); a 64-byte load split across cache lines is what Intel penalizes 2x -let STAGGER = 1088l // plane starts differ in their low 12 address bits (1088 = 1024 + 64 keeps six starts distinct mod 4096): planes that alias in the L1/L2 set logic make a run's time depend on where the heap put them +let ALIGN = 64l // a 64-byte load split across two cache lines is the 2x penalty the fixture must not introduce +let STAGGER = 1088l // 1024 + 64: keeps the six plane starts distinct in their low 12 address bits let SCALE_BYTE = uint8(0x3C) // as f16 0x3C3C = 1.06, as f32 0x3C3C3C3C = 0.0115, as e8m0 2^-67, as a 6-bit sub-scale 60 - normal in every scale form -//! every plane of one format in ONE arena at fixed offsets: the heap places separate arrays at -//! run-dependent relative addresses, and k6 decode read 3.9-6.0 ms across identical runs from that alone +//! every plane of one format in ONE arena at fixed offsets struct Planes { arena : array - kq : int64 // the gemv's weight plane - tkq : int64 // the tile's weight plane (the unpacked panel form where the format has one) + gemvKq : int64 // the gemv's weight plane + tileKq : int64 // the tile's weight plane (the unpacked panel form where the format has one) ks : int64 // scale plane xq : int64 // int8 activations xs : int64 // f32 activation scales @@ -204,10 +203,11 @@ def arena_base(var p : Planes) : int64 { } } +[arch(at = "../../ARCHITECTURE_MEASUREMENT.md#cpu-kernel-bench-fixture")] def fill_planes(var p : Planes; sh : PlaneShape; n, d, ntok : int64) { var end = 0l - p.kq = next_plane(end, d * sh.wq_row, 0) - p.tkq = next_plane(end, d * sh.tq_row, 1) + p.gemvKq = next_plane(end, d * sh.gemvW_row, 0) + p.tileKq = next_plane(end, d * sh.tileW_row, 1) p.ks = next_plane(end, d * sh.ws_row, 2) p.xq = next_plane(end, ntok * n, 3) p.xs = next_plane(end, ntok * n / 32l * 4l, 4) @@ -216,24 +216,24 @@ def fill_planes(var p : Planes; sh : PlaneShape; n, d, ntok : int64) { p.arena |> reserve(end + BASE_ALIGN + BASE_OFFSET) p.arena |> resize(end + BASE_ALIGN + BASE_OFFSET) let base = arena_base(p) - p.kq += base - p.tkq += base + p.gemvKq += base + p.tileKq += base p.ks += base p.xq += base p.xs += base p.xbs += base p.y += base var seed = 0x1234u - for (i in range64(p.kq, p.kq + d * sh.wq_row)) { + for (i in range64(p.gemvKq, p.gemvKq + d * sh.gemvW_row)) { seed = seed * 1664525u + 1013904223u p.arena[i] = uint8(seed >> 24u) } - for (i in range64(p.tkq, p.tkq + d * sh.tq_row)) { + for (i in range64(p.tileKq, p.tileKq + d * sh.tileW_row)) { seed = seed * 1664525u + 1013904223u p.arena[i] = uint8(seed >> 24u) } for (i in range64(p.ks, p.ks + d * sh.ws_row)) { - p.arena[i] = SCALE_BYTE // scale planes hold normal numbers in every scale form, never random bytes: Intel runs denormal math ~100x slower + p.arena[i] = SCALE_BYTE } for (i in range64(p.xq, p.xq + ntok * n)) { seed = seed * 1664525u + 1013904223u @@ -306,6 +306,7 @@ def team_gemv_mx4(fn : GemvMx4Fn; var yp : float?; wn, we : uint8 const?; xqp : //! the team arm's interleaved timing: the block dispatches row `row` the engine's way and every lane adds its //! call time to laneP[slot]; the row keeps the wall (best, median) and the lanes' fastest and slowest sums +[arch(at = "../../ARCHITECTURE_MEASUREMENT.md#cpu-kernel-bench-fixture")] def time_rows_team(var stats : array; rounds : int; blk : block<(row : int; var laneP : int64?) : void>) { var laneUs : array laneUs |> resize(get_total_hw_jobs() + 1) @@ -313,7 +314,7 @@ def time_rows_team(var stats : array; rounds : int; blk : block<(row : for (row in range(length(stats))) { stats[row].us |> reserve(rounds) stats[row].lane_lo = 999999999l - for (_w in range(TEAM_BURN)) { // codegen, then a burn: the first format's team rows read one slot at 8 ms without it + for (_w in range(TEAM_BURN)) { invoke(blk, row, laneP) } } @@ -382,10 +383,11 @@ def perm_wanted(spec, suffix : string) : bool { } //! interleaved timing: round r calls every selected row once before round r + 1 starts +[arch(at = "../../ARCHITECTURE_MEASUREMENT.md#cpu-kernel-bench-fixture")] def time_rows(var stats : array; rounds : int; blk : block<(row : int) : void>) { for (row in range(length(stats))) { stats[row].us |> reserve(rounds) - for (_w in range(3)) { // codegen + warm-up, unmeasured: Intel k6 ramps for ~3 rounds (3425, 1989, then steady 1637 us) - one warm call is not enough + for (_w in range(3)) { invoke(blk, row) } } @@ -402,7 +404,7 @@ def time_rows(var stats : array; rounds : int; blk : block<(row : int) def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; var p : Planes; nsb : int64; sh : PlaneShape) { let n = int64(cfg.n) let d = int64(cfg.d) - let plane_bytes = int64(cfg.d) * (sh.wq_row + sh.ws_row) + let plane_bytes = int64(cfg.d) * (sh.gemvW_row + sh.ws_row) var vs <- gemv_variants(f) var stats : array var fns : array @@ -413,7 +415,7 @@ def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va } } unsafe { - let kqp = addr(p.arena[p.kq]) + let kqp = addr(p.arena[p.gemvKq]) let ksp = addr(p.arena[p.ks]) let xqp = addr(p.arena[p.xq]) let xsp = addr(p.arena[p.xs]) @@ -429,13 +431,13 @@ def bench_gemv_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va } } if (cfg.tsv) { // the row's answer, one solo call each: modes and seats must agree here before their times are compared - for (row in range(length(fns))) { - invoke(fns[row], yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) + for (fn, st in fns, stats) { + invoke(fn, yp, kqp, ksp, xqp, xsp, xbsp, n, 0l, d) var acc = 0.0lf for (i in range64(d)) { acc += double(yp[i]) } - print("# ysum\t{name}\t{stats[row].perm}\t{acc}\t{yp[0]}\t{yp[d - 1l]}\n") + print("# ysum\t{name}\t{st.perm}\t{acc}\t{yp[0]}\t{yp[d - 1l]}\n") } } } @@ -451,9 +453,9 @@ def bench_tile_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va let n = int64(cfg.n) let d = int64(cfg.d) let ntok = int64(cfg.ntok) - let qrow = sh.tq_row + let qrow = sh.tileW_row let srow = sh.ws_row - let plane_bytes = int64(cfg.d) * (sh.tq_row + sh.ws_row) + let plane_bytes = int64(cfg.d) * (sh.tileW_row + sh.ws_row) var mrs <- companion_ints(layout_variants(f)) var vs <- tile_variants(f) var stats : array @@ -473,7 +475,7 @@ def bench_tile_kq(var rows : int&; cfg : BenchArgs; name : string; f : KqFmt; va mrOf |> push(mr) } unsafe { - let kqp = addr(p.arena[p.tkq]) + let kqp = addr(p.arena[p.tileKq]) let ksp = addr(p.arena[p.ks]) let xqp = addr(p.arena[p.xq]) let xsp = addr(p.arena[p.xs]) @@ -578,19 +580,19 @@ def q8_tile_rows(cfg : BenchArgs; mx4 : bool; d : int64; var stats : array(p.arena[p.kq]) + let kqp = addr(p.arena[p.gemvKq]) let ksp = addr(p.arena[p.ks]) let xqp = addr(p.arena[p.xq]) let xsp = addr(p.arena[p.xs]) @@ -645,11 +647,11 @@ def bench_q8_family(var rows : int&; cfg : BenchArgs; mx4 : bool; var p : Planes var t0k = 0l while (t0k < ntok) { if (mx4) { - invoke(fnsM[row], yp, kqp + g * mr * sh.tq_row, ksp + g * mr * sh.ws_row, xqp, xsp, xbsp, n, d, g, t0k) + invoke(fnsM[row], yp, kqp + g * mr * sh.tileW_row, ksp + g * mr * sh.ws_row, xqp, xsp, xbsp, n, d, g, t0k) } elif (g_s16) { - invoke(fnsS[row], yp, reinterpret(kqp + g * mr * sh.tq_row), reinterpret(ksp + g * mr * sh.ws_row), xqp, xsp, xbsp, n, d, g, t0k) + invoke(fnsS[row], yp, reinterpret(kqp + g * mr * sh.tileW_row), reinterpret(ksp + g * mr * sh.ws_row), xqp, xsp, xbsp, n, d, g, t0k) } else { - invoke(fnsQ[row], yp, reinterpret(kqp + g * mr * sh.tq_row), reinterpret(ksp + g * mr * sh.ws_row), xqp, xsp, xbsp, n, d, g, t0k) + invoke(fnsQ[row], yp, reinterpret(kqp + g * mr * sh.tileW_row), reinterpret(ksp + g * mr * sh.ws_row), xqp, xsp, xbsp, n, d, g, t0k) } t0k += stepOf[row] } diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index 126754bf66..a3e7d17295 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -11,7 +11,6 @@ require llvm/daslib/llvm_jit_code require llvm/daslib/llvm_jit_common require daslib/ast_boost require dasllama/dasllama_gemm_schema -require daslib/fio require llvm/daslib/llvm_tune // The perm-parameterized Q8 GEMM tile generator (gemm_generator_plan.md part 3, promoted from @@ -224,8 +223,8 @@ def private perm_declines(var gc : LlvmCodeCtx; p : TilePerm) : bool { // nolint if ((p.dotPrim != "maddubs" && p.dotPrim != "vpdpbusd" && p.dotPrim != "vpdpbssd") || (!g_target_is_x64) || (p.width != 256 && p.width != 512) + || (p.grid == "vbmi" && (p.width != 512 || p.dotPrim == "vpdpbssd" || !g_target_x64_avx512vbmi)) // the lattice is zmm VBMI || (p.dotPrim == "vpdpbssd" && p.width != 256)) return true // 512-bit bssd is AVX10 — unproven - if (p.grid == "vbmi" && (p.width != 512 || p.dotPrim == "vpdpbssd" || !g_target_x64_avx512vbmi)) return true // the lattice is zmm VBMI if (p.width == 256) { if ((p.dotPrim == "maddubs" && !g_target_x64_avx2) || (p.dotPrim == "vpdpbusd" && !g_target_x64_vnni256) @@ -260,8 +259,8 @@ struct private TileEmit { abs_decl : LLVMOpaqueValue? // x64 sign trick: llvm.abs (VPABSB) abs_ty : LLVMOpaqueType? psign_decl : LLVMOpaqueValue? // x64 width-256 sign apply (VPSIGNB) - iq3s_panel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) - iq3s_grid_g : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) + gridPanel : LLVMOpaqueValue? // iq3s/iq3xxs gemv: the per-superblock alloca panel (null = tile mode, the caller unpacked) + gridGlobal : LLVMOpaqueValue? // iq3s gemv: the [512 x i32] grid global (iq3xxs: its [256 x i32] halved grid) sign_code_pm1_g : LLVMOpaqueValue? // the row-group grid decode: ksigns code -> eight +-1 bytes, [128 x i64], parity bit folded in gridRows : bool // set while the row-group grid decode emits: the column dword read (a panel-form win) reverts to byte loads vbmi : bool // grid = "vbmi": the grid gemv decodes through the VBMI symbol lattice (row form) @@ -450,9 +449,6 @@ def private bias_acc_init(var te : TileEmit; var bi, xlo, xhi : LLVMOpaqueValue? return LLVMBuildShuffleVector(b, te.types, one, one, mask, "binit{tk}") } -// One 32-k block for tokens [tokBase, tokBase+tokCount): 8*rq shared weight vectors x tokens -// lane-dots at full width, then the per-block scale fold — the exact op order of -// dot_q8q8_laneq4x4_template, so fast-math contraction sees the same input across emitters. //! llvm.prefetch(read, high locality, data) of the weight plane at byte offset `off` def private prefetch_wg(var te : TileEmit; var off : LLVMOpaqueValue?) { let b = te.builder @@ -461,6 +457,9 @@ def private prefetch_wg(var te : TileEmit; var off : LLVMOpaqueValue?) { LLVMBuildCall2(b, te.prefetch_ty, te.prefetch_decl, args, "") } +// One 32-k block for tokens [tokBase, tokBase+tokCount): 8*rq shared weight vectors x tokens +// lane-dots at full width, then the per-block scale fold — the exact op order of +// dot_q8q8_laneq4x4_template, so fast-math contraction sees the same input across emitters. def private emit_block(var te : TileEmit; var bi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { let b = te.builder let rq = te.rq @@ -998,17 +997,10 @@ def private or_bit_x10(te : TileEmit; var w, bytes, maskv : LLVMOpaqueValue?; hi return LLVMBuildOr(b, w, sel, name) } -//! a 64-byte-aligned alloca at the kernel's entry: LLVM then realigns the frame, so every ymm/zmm spill slot sits -//! inside one cache line whatever RSP the caller arrives with (Intel splits a line-straddling 64-byte access; the -//! per-process stack phase was k6's 4.0 / 5.4 ms draw on zen2 and its normal-vs-tune mode gap on Granite Rapids) -def private frame_align64(b : LLVMOpaqueBuilder?; types : PrimitiveTypes?) { - var a = LLVMBuildAlloca(b, LLVMArrayType(types.t_int8, 64u), "frame64") - LLVMSetAlignment(a, 64u) -} - // One 256-weight SUPERBLOCK, K-quant grp form (te.kq set): weight vectors unpacked // ONCE per (sub-block, dword-group) and dotted against every token — weight-stationary, // bit-exact vs per-token GEMVs (tokCount=1). kq v2 folds integer sub-scales/bsums against Q8_K-form activations. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f : LLVMOpaqueValue? [8]; tokBase, tokCount : int) { // nolint:STYLE037,STYLE038 — one register-coupled superblock loop nest; the k-quant decode arms share live values let b = te.builder let rq = te.rq @@ -1034,7 +1026,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f // x64 dot kinds take the activation dword straight from memory (vpbroadcastd, see kq_dot_mem); NEON keeps chunk loads let memBcast = te.dotKind != DOT_SDOT && te.dotKind != DOT_SMMLA let madd16 = memBcast && te.dotKind == DOT_MADDUBS - let fuseScale = madd16 && (k63 || te.kq == 5) // the sub-scale rides the chain flush (pmaddwd), no i32 multiply, no a0/a1; k4 measured 4% slower fused (one flush per sub-block) + let fuseScale = madd16 && (k63 || te.kq == 5) // the sub-scale rides the chain flush (pmaddwd): no i32 multiply, no a0/a1; k4 stays unfused - one flush per sub-block loses more than the fold saves var vri8 = LLVMVectorType(te.types.t_int8, uint(te.rv)) // weight-side superblock scale rows: k4/k5 f16 d/dmin; k6/k3 f16 d only; q40 has none (per-32 d loads live below) var dv : LLVMOpaqueValue? [2] @@ -1121,7 +1113,6 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f var wlo : LLVMOpaqueValue? var whi : LLVMOpaqueValue? if (k3) { - // k3: qs columns 2blk / 2blk + 1 hold this sub-block's four j sites at 2j, hmask column blk its eight sites (lo at j, hi at 4 + j) - three loads per sub-block, nothing lives past it var q3l = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 2) * 4 * mr + qd * w8)), ""), "q3l{blk}_{qd}") var q3h = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((blk * 2 + 1) * 4 * mr + qd * w8)), ""), "q3h{blk}_{qd}") var h3 = load_vec(te, te.vwi8, te.wg, LLVMBuildAdd(b, wb, te.types->ConstI64(uint64((64 + blk * 4) * mr + qd * w8)), ""), "h3{blk}_{qd}") @@ -1154,7 +1145,6 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f wlo = or_bit_x10(te, wlo, hb, maskLo, false, "wlo5{blk}_{j * rq + qd}") whi = or_bit_x10(te, whi, hb, maskHi, true, "whi5{blk}_{j * rq + qd}") } elif (k6 && !te.kqBytes) { - // the plane's qh columns 2blk / 2blk + 1 hold this sub-block's four j sites at 2j: two loads per sub-block, nothing lives past it let hbase = 128 * mr + blk * 2 * 4 * mr var hloOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + qd * w8)), "") var hhiOff = LLVMBuildAdd(b, wb, te.types->ConstI64(uint64(hbase + 4 * mr + qd * w8)), "") @@ -1215,7 +1205,7 @@ def private emit_block_kqv2(var te : TileEmit; var sbi : LLVMOpaqueValue?; var f } } } - if (madd16 && wholeSub) { // one widen per (token, qd, chain) for the whole sub-block + if (madd16 && wholeSub) { for (k in range(tokCount * rq)) { if (fuseScale) { iacc[k] = madd16_flush_scaled(te, iacc[k], p16lo[k], s16lo[k % rq]) @@ -1378,7 +1368,7 @@ def private iq3s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid + te.gridGlobal = grid } // The five grid formats' per-dword decode - the raw grid word of the row whose packed grp region @@ -1398,8 +1388,9 @@ def private wg_col(mr, pos : int) : int => (pos / 4) * mr * 4 + pos % 4 //! the byte at packed position `pos`, read as one dword load of its column with the byte split off in //! registers - the four positions of a column share the load through CSE +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#grid-decode-forms")] def private wg_col_byte(var te : TileEmit; var base : LLVMOpaqueValue?; pos : int) : LLVMOpaqueValue? { - if (te.gridRows) return wg_byte(te, base, wg_col(te.interleave, pos)) // the row form measured 1.8x slower on the dword read (zen4 iq2xs / iq3xxs) + if (te.gridRows) return wg_byte(te, base, wg_col(te.interleave, pos)) let b = te.builder var col = wg_i32(te, base, (pos / 4) * te.interleave * 4) let sh = 8 * (pos % 4) @@ -1409,7 +1400,7 @@ def private wg_col_byte(var te : TileEmit; var base : LLVMOpaqueValue?; pos : in //! grid[gidx] - a raw magnitude word; the block emitter applies the signs per vector def private grid_word(var te : TileEmit; var gidx : LLVMOpaqueValue?) : LLVMOpaqueValue? { - var gp = LLVMBuildGEP2(te.builder, te.types.t_int32, te.iq3s_grid_g, gidx, "") + var gp = LLVMBuildGEP2(te.builder, te.types.t_int32, te.gridGlobal, gidx, "") return LLVMBuildLoad2Aligned(te.builder, te.types.t_int32, gp, 4u, "") } @@ -1424,7 +1415,7 @@ def private grid_pair_half(var te : TileEmit; var idx2 : LLVMOpaqueValue?; h : i //! the u64 grid's word pair at even index `idx2` as one i64 - a row's 8 weights in one load def private grid_pair_i64(var te : TileEmit; var idx2 : LLVMOpaqueValue?) : LLVMOpaqueValue? { let b = te.builder - var gp = LLVMBuildGEP2(b, te.types.t_int64, te.iq3s_grid_g, LLVMBuildLShr(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), ""), "") + var gp = LLVMBuildGEP2(b, te.types.t_int64, te.gridGlobal, LLVMBuildLShr(b, idx2, LLVMConstInt(te.types.t_int32, 1ul, 0), ""), "") return LLVMBuildLoad2Aligned(b, te.types.t_int64, gp, 4u, "") } @@ -1438,13 +1429,13 @@ def private decode_iq3s_dword(var te : TileEmit; var base : LLVMOpaqueValue?; bl return grid_word(te, LLVMBuildOr(b, qv, hb, "")) } -//! iq3xxs: an 8-bit index into the halved grid (the column read measured 7862 -> 6602 us; byte loads stay for iq3s/iq2s/iq2xxs, where it lost) +//! iq3xxs: an 8-bit index into the halved grid, read through its column dword def private decode_iq3xxs_dword(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l, h : int) : LLVMOpaqueValue? { return grid_word(te, wg_col_byte(te, base, blk * 8 + 2 * l + h)) } -//! iq2xs: row `base`'s u16 word for (blk, l) - low 9 bits the grid index, the top 7 the ksigns code -//! (the column read measured 6501 -> 4702 us; one u16 load measured slower than two bytes) +//! iq2xs: row `base`'s u16 word for (blk, l) - low 9 bits the grid index, the top 7 the ksigns code, +//! read as its two column bytes def private iq2xs_word16(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l : int) : LLVMOpaqueValue? { let b = te.builder let lp = (blk * 4 + l) * 2 @@ -1500,19 +1491,14 @@ def private sign_col_pm1(var te : TileEmit; var col : LLVMOpaqueValue?; r0, l : return LLVMBuildOr(b, m, splat_i8w(te, 1), name) } -//! the sdot lattice composes row groups straight from the grid words; on x86 the panel's latency chain does not -//! scale with the core, so x86-vnni512 takes the row form for iq2s and iq2xxs (zen4: 4918 -> 3600, 4866 -> 4300 us) -//! and the panel elsewhere (iq2xs / iq3xxs lose 1.8x, zen2 loses on all five); DASLLAMA_GRID_ROWS_X86=1 / 0 forces it (clear .jitted_scripts between runs) +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#grid-decode-forms")] def private grid_rows_path(te : TileEmit) : bool { if (te.vbmi) return true if (te.dotKind == DOT_SDOT) return te.rv == 4 if (te.dotKind != DOT_MADDUBS && te.dotKind != DOT_VPDPBUSD && te.dotKind != DOT_VPDPBSSD) return false - let knob = get_env_variable("DASLLAMA_GRID_ROWS_X86") - if (knob == "1") return true - if (knob == "0") return false let cls = tune_cpu_class() - if (cls == "x86-vnni512") return te.kq == 25 // zen4: iq2xxs (iq2s and iq3xxs win alone, lose at 16 lanes) - if (cls == "x86-amx") return te.kq == 25 || te.kq == 34 // Granite Rapids: iq2xxs 1.47x, iq3xxs 1.36x at the engine shape + if (cls == "x86-vnni512") return te.kq == 25 + if (cls == "x86-amx") return te.kq == 25 || te.kq == 34 return false } @@ -1639,7 +1625,7 @@ def private emit_kq_gather(var te : TileEmit; var sbi : LLVMOpaqueValue?) { for (l in range(4)) { for (h in range(2)) { var w = decode_kq_dword(te, base, blk, l, h) - var dp = LLVMBuildGEP2(b, te.types.t_int8, te.iq3s_panel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(panel_group_off(mr, blk, l, h))), ""), "") + var dp = LLVMBuildGEP2(b, te.types.t_int8, te.gridPanel, LLVMBuildAdd(b, rT4, te.types->ConstI64(uint64(panel_group_off(mr, blk, l, h))), ""), "") LLVMBuildStore(b, w, dp) } } @@ -1669,7 +1655,7 @@ def private iq3xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid + te.gridGlobal = grid } @@ -1689,7 +1675,7 @@ def private iq2s_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid + te.gridGlobal = grid } @@ -1709,7 +1695,7 @@ def private iq2xs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid + te.gridGlobal = grid } // The iq2xxs gemv gather global: the u64 grid as a [512 x i32] private constant (low/high word pairs). @@ -1728,7 +1714,7 @@ def private iq2xxs_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { LLVMSetLinkage(grid, LLVMLinkage.LLVMPrivateLinkage) LLVMSetAlignment(grid, 64u) } - te.iq3s_grid_g = grid + te.gridGlobal = grid } @@ -1799,6 +1785,7 @@ def private row_weights8(var te : TileEmit; var base : LLVMOpaqueValue?; blk, l //! the lattice's constant tables, emitted once per module: the grid's two compact code planes (entry e's //! low / high half - four 2-bit symbols for the iq2 alphabet, two 3-bit symbols for iq3), the alphabet as a //! per-lane vpshufb table, and ksigns_iq2xs whole (128 bytes: exactly the two registers one VPERMI2B indexes) +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#vbmi-lattice")] def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { let tag = vbmi_tag(te.kq) te.vb_plane[0] = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.p0") @@ -1806,23 +1793,12 @@ def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { te.vb_alpha = LLVMGetNamedGlobal(mod, "dasllama.vbmi.{tag}.alpha") te.vb_ksigns = LLVMGetNamedGlobal(mod, "dasllama.vbmi.ksigns") if (te.vb_plane[0] != null && te.vb_plane[1] != null && te.vb_alpha != null && te.vb_ksigns != null) return - let iq3 = te.kq == 33 || te.kq == 34 - var alpha : array - if (te.kq == 34) { - alpha <- [2, 6, 10, 14, 18, 22, 26, 31] - } elif (te.kq == 33) { - alpha <- [1, 3, 5, 7, 9, 11, 13, 15] - } else { - alpha <- [8, 25, 43] - } + let alpha <- vbmi_alphabet(te.kq) let entries = vbmi_entries(te.kq) var p0 : array var p1 : array p0 |> reserve(entries) p1 |> reserve(entries) - // one grid table per helper frame: a das frame holds every local of its function, and five by-value grid - // tables in one frame (~17 KB) overflowed the codegen context's stack into the heap (cold-start SIGSEGV - // in LLVMContextDispose); the existing global builders hold one 8 KB table each, and so do these if (te.kq == 34) { vbmi_planes_iq3xxs(te, p0, p1, alpha) } elif (te.kq == 33) { @@ -1854,14 +1830,14 @@ def private vbmi_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { } //! an iq3 entry is one word of 4 weights: weights 0-1 and 2-3 per plane -def private vbmi_planes_iq3xxs(var te : TileEmit; var p0, p1 : array; alpha : array) { +def private vbmi_planes_iq3xxs(te : TileEmit; var p0, p1 : array; alpha : array) { let g = iq3xxs_grid() for (e in range(256)) { vbmi_push_iq3(te, p0, p1, g[e], alpha) } } -def private vbmi_planes_iq3s(var te : TileEmit; var p0, p1 : array; alpha : array) { +def private vbmi_planes_iq3s(te : TileEmit; var p0, p1 : array; alpha : array) { let g = iq3s_grid() for (e in range(512)) { vbmi_push_iq3(te, p0, p1, g[e], alpha) @@ -1869,37 +1845,63 @@ def private vbmi_planes_iq3s(var te : TileEmit; var p0, p1 : array; alpha : array) { +def private vbmi_planes_iq2xxs(te : TileEmit; var p0, p1 : array; alpha : array) { let g = iq2xxs_grid2() for (e in range(256)) { vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) } } -def private vbmi_planes_iq2xs(var te : TileEmit; var p0, p1 : array; alpha : array) { +def private vbmi_planes_iq2xs(te : TileEmit; var p0, p1 : array; alpha : array) { let g = iq2xs_grid2() for (e in range(512)) { vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) } } -def private vbmi_planes_iq2s(var te : TileEmit; var p0, p1 : array; alpha : array) { +def private vbmi_planes_iq2s(te : TileEmit; var p0, p1 : array; alpha : array) { let g = iq2s_grid2() for (e in range(1024)) { vbmi_push_iq2(te, p0, p1, g[2 * e], g[2 * e + 1], alpha) } } -def private vbmi_push_iq3(var te : TileEmit; var p0, p1 : array; word : uint; alpha : array) { +def private vbmi_push_iq3(te : TileEmit; var p0, p1 : array; word : uint; alpha : array) { p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(word, alpha, 3, 2)), 0)) p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(word >> 16u, alpha, 3, 2)), 0)) } -def private vbmi_push_iq2(var te : TileEmit; var p0, p1 : array; w0, w1 : uint; alpha : array) { +def private vbmi_push_iq2(te : TileEmit; var p0, p1 : array; w0, w1 : uint; alpha : array) { p0 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(w0, alpha, 2, 4)), 0)) p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(w1, alpha, 2, 4)), 0)) } +//! the grid's symbol alphabet: the halved iq3xxs magnitudes, iq3s's odd magnitudes, the iq2 family's three +def private vbmi_alphabet(kq : int) : array { + if (kq == 34) return <- [2, 6, 10, 14, 18, 22, 26, 31] + if (kq == 33) return <- [1, 3, 5, 7, 9, 11, 13, 15] + return <- [8, 25, 43] +} + +//! the grid gemv's module-level tables: the format's grid, the ksigns +-1 table when the row form reads it, the lattice's +def private grid_emit_globals(var te : TileEmit; fmt : int; mod : LLVMOpaqueModule?) { + if (fmt == 33) { + iq3s_emit_globals(te, mod) + } elif (fmt == 23) { + iq2s_emit_globals(te, mod) + } elif (fmt == 24) { + iq2xs_emit_globals(te, mod) + } elif (fmt == 25) { + iq2xxs_emit_globals(te, mod) + } else { + iq3xxs_emit_globals(te, mod) + } + sign_pm1_emit_globals(te, mod) + if (te.vbmi) { + vbmi_emit_globals(te, mod) + } +} + def private vbmi_tag(kq : int) : string => kq == 25 ? "iq2xxs" : (kq == 24 ? "iq2xs" : (kq == 23 ? "iq2s" : (kq == 34 ? "iq3xxs" : "iq3s"))) //! grid entries per format: the compact plane's byte count @@ -1919,7 +1921,7 @@ def private vbmi_pack_word(word : uint; alpha : array; bits, nbytes : int) return code } -def private vbmi_const_bytes(var te : TileEmit; mod : LLVMOpaqueModule?; name : string; var elems : array) : LLVMOpaqueValue? { +def private vbmi_const_bytes(te : TileEmit; mod : LLVMOpaqueModule?; name : string; var elems : array) : LLVMOpaqueValue? { var g = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int8, uint(length(elems))), name) LLVMSetInitializer(g, LLVMConstArray(te.types.t_int8, array_data_ptr(elems), uint(length(elems)))) LLVMSetGlobalConstant(g, 1) @@ -1954,24 +1956,16 @@ def private vbmi_mshift(var te : TileEmit; var v : LLVMOpaqueValue?; ctrl : arra //! 64 indices through a byte table of `entries` (256 / 512 / 1024): VPERMI2B covers 128 entries per pair of //! table registers, index bit 7 blends the pairs, and the caller's masks (the index's 9th and 10th bits, per //! lane) blend the 256-entry halves above that -def private vbmi_lookup(var te : TileEmit; var tbl : LLVMOpaqueValue?; entries : int; var idx : LLVMOpaqueValue?; var masks : array; name : string) : LLVMOpaqueValue? { +def private vbmi_lookup(var te : TileEmit; var tbl : LLVMOpaqueValue?; entries : int; var idx : LLVMOpaqueValue?; var masks : array; name : string) : LLVMOpaqueValue? { // nolint:LINT014 — var for non-const value handles: a const element cannot feed LLVMBuildSelect let b = te.builder - var level : array - for (k in range(entries / 128)) { - level |> push(vbmi_permi2b(te, vbmi_table_part(te, tbl, 2 * k, "{name}.t{2 * k}"), idx, vbmi_table_part(te, tbl, 2 * k + 1, "{name}.t{2 * k + 1}"), "{name}.l{k}")) - } + var level <- [for (k in range(entries / 128)); vbmi_permi2b(te, vbmi_table_part(te, tbl, 2 * k, "{name}.t{2 * k}"), idx, vbmi_table_part(te, tbl, 2 * k + 1, "{name}.t{2 * k + 1}"), "{name}.l{k}")] var high = LLVMBuildICmp(b, LLVMIntPredicate.LLVMIntSLT, idx, LLVMConstNull(te.vwi8), "{name}.b7") var depth = 0 while (length(level) > 1) { var sel = depth == 0 ? high : masks[depth - 1] - var nxt : array - for (j in range(length(level) / 2)) { - nxt |> push(LLVMBuildSelect(b, sel, level[2 * j + 1], level[2 * j], "{name}.s{depth}_{j}")) - } + var nxt <- [for (j in range(length(level) / 2)); LLVMBuildSelect(b, sel, level[2 * j + 1], level[2 * j], "{name}.s{depth}_{j}")] level |> clear() - for (v in nxt) { - level |> push(v) - } + level |> push_from(nxt) depth++ } return level[0] @@ -2016,13 +2010,7 @@ def private vbmi_ksigns(var te : TileEmit; var codes : LLVMOpaqueValue?; name : return vbmi_permi2b(te, ks0, LLVMBuildAnd(te.builder, codes, splat_i8w(te, 127), ""), ks1, name) } -//! One block of a grid format as ROW GROUPS through the VBMI symbol lattice. Per format the block's index -//! bytes are gathered into one 64-lane vector per column (lane r*4 + position), looked up in the compact code -//! planes (vbmi_lookup; the 9th / 10th index bits arrive as lane masks), and the row's sign bytes land in the -//! same lane layout (the plane's own column for iq3s / iq2s, one ksigns VPERMI2B over the 7-bit codes for the -//! rest). Per row group and weight octet a constant two-source shuffle places each row's code bytes in its -//! qword, VPMULTISHIFTQB spreads the symbols into bytes, one vpshufb maps them to magnitudes; the signs ride the -//! activation copy as a mask. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#vbmi-lattice")] def private emit_grid_rows_blk_vbmi(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { // nolint:STYLE037,STYLE038 — one block's decode: the five formats' index assembly and the shared expansion let b = te.builder let n = te.width / 64 @@ -2116,6 +2104,7 @@ def private emit_grid_rows_blk_vbmi(var te : TileEmit; var wbP, xb : LLVMOpaqueV //! One block of a grid format's decode as ROW GROUPS: a weight-width vector is width/64 rows x 8 weights //! straight from the grid words (one u64 grid entry per iq2 row), signed by +-1 bytes - the ksigns table row //! per code or the plane's sign column (iq3s/iq2s); one pairwise add per group pair yields the fold's per-row lanes. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#grid-decode-forms")] def private emit_grid_rows_blk(var te : TileEmit; var wbP, xb : LLVMOpaqueValue?; blk : int; var signCol : LLVMOpaqueValue? [2]; var a, a1 : LLVMOpaqueValue? [8]; iq2 : bool) { let b = te.builder let n = te.width / 64 @@ -2171,7 +2160,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var let panel = te.kq == 33 || te.kq == 34 || te.kq == 23 || te.kq == 24 || te.kq == 25 // iq3s/iq3xxs/iq2s/iq2xs/iq2xxs: the byte-expanded signed panel replaces the nibble+LUT decode let iq2 = te.kq == 23 || te.kq == 24 // iq2s/iq2xs: per-16 UNSIGNED strips - lo/hi halves keep split accumulators let nl = te.kq == 45 // iq4nl: the 44 LUT decode with q40's PER-BLOCK f16 d fold (no sub-scales, no min term) - let gather = panel && te.iq3s_panel != null // gemv mode: gather this superblock off the packed planes first + let gather = panel && te.gridPanel != null // gemv mode: gather this superblock off the packed planes first // the decode shape composes row GROUPS straight from the grid words (emit_grid_rows_blk) - no panel let pairs = gather && tokCount == 1 && grid_rows_path(te) var wbase = te.wg @@ -2182,7 +2171,7 @@ def private emit_block_iq4xs(var te : TileEmit; var sbi : LLVMOpaqueValue?; var if (!pairs) { emit_kq_gather(te, sbi) } - wbase = te.iq3s_panel + wbase = te.gridPanel wb = te.types->ConstI64(0ul) } else { wb = LLVMBuildMul(b, sbi, te.types->ConstI64(uint64(mr * (panel ? 256 : 128))), "wb") @@ -2757,7 +2746,6 @@ def private emit_amx_tile(var gc : LlvmCodeCtx; p : TilePerm; f16s : bool) : boo var after = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "after") LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) var yp = LLVMGetParam(gc.impl, 0u) var wg = LLVMGetParam(gc.impl, 1u) var sg = LLVMGetParam(gc.impl, 2u) @@ -2905,7 +2893,6 @@ def private emit_amx_tile_pipelined(var gc : LlvmCodeCtx; p : TilePerm; f16s : b var after = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "after") LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) var yp = LLVMGetParam(gc.impl, 0u) var wg = LLVMGetParam(gc.impl, 1u) var sg = LLVMGetParam(gc.impl, 2u) @@ -3059,7 +3046,6 @@ def private tile_gen_impl(var gc : LlvmCodeCtx; f16s : bool) : bool { // entry: per-token activation/scale bases; nb = n / 32 LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = p.kstep) sa.yp = LLVMGetParam(gc.impl, 0u) te.wg = LLVMGetParam(gc.impl, 1u) @@ -3171,7 +3157,6 @@ def private witness_gen(var gc : LlvmCodeCtx) : bool { var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") let b = gc.jit.builder LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) if (dot_kind(p.dotPrim) == DOT_AMX) { // TMUL executes only after the per-process XTILEDATA grant, so the arch_prctl syscall IS the witness let mod = LLVMGetGlobalParent(gc.impl) @@ -3206,7 +3191,6 @@ def private cfg_gen(var gc : LlvmCodeCtx) : bool { var entry = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "entry") let b = gc.jit.builder LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) if (dot_kind(p.dotPrim) == DOT_AMX && p.latch != 0) { let mod = LLVMGetGlobalParent(gc.impl) var glob = LLVMGetNamedGlobal(mod, "dasllama.amx.tilecfg") @@ -3270,7 +3254,6 @@ def private gemv_gen_impl(var gc : LlvmCodeCtx; f16s : bool) : bool { // entry: group range off the row range; d=0/t0=0 turn emit_slice's store offset into the absolute row g*mr LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = p.gkstep) sa.yp = LLVMGetParam(gc.impl, 0u) let wp = LLVMGetParam(gc.impl, 1u) @@ -3338,7 +3321,6 @@ def private mx4_gemv_gen(var gc : LlvmCodeCtx) : bool { // entry: group range off the row range; plane strides are nibble n/2 and exponent n/32 LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = p.gkstep) sa.yp = LLVMGetParam(gc.impl, 0u) let wn = LLVMGetParam(gc.impl, 1u) @@ -3399,7 +3381,6 @@ def private mx4_tile_gen(var gc : LlvmCodeCtx) : bool { // entry: per-token activation/scale bases; wn/we arrive pre-offset to the group LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = p.kstep) sa.yp = LLVMGetParam(gc.impl, 0u) te.wg = LLVMGetParam(gc.impl, 1u) @@ -3459,28 +3440,13 @@ def private kq_gemv_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: group range off the row range; strides are disk footprints (quants 128/160/192B, scales 20/20/18B) LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) if (fmt == 33 || fmt == 34 || fmt == 23 || fmt == 24 || fmt == 25) { // iq3s/iq3xxs/iq2s/iq2xs: the gemv gathers each superblock into an alloca panel (the tile gets the runtime unpack) te.gfn = gc.impl te.gctx = gc.jit.ctx - if (fmt == 33) { - iq3s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } elif (fmt == 23) { - iq2s_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } elif (fmt == 24) { - iq2xs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } elif (fmt == 25) { - iq2xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } else { - iq3xxs_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } - sign_pm1_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - if (te.vbmi) { - vbmi_emit_globals(te, LLVMGetGlobalParent(gc.impl)) - } - te.iq3s_panel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "iq3s.panel") - LLVMSetAlignment(te.iq3s_panel, 64u) // a cache line: at 32 half the width-512 panel loads straddled one + grid_emit_globals(te, fmt, LLVMGetGlobalParent(gc.impl)) + te.gridPanel = LLVMBuildAlloca(b, LLVMArrayType(gc.jit.types.t_int8, uint(256 * te.interleave)), "grid.panel") + LLVMSetAlignment(te.gridPanel, 64u) // a cache line: at 32, half the width-512 panel loads straddle one } var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) @@ -3550,7 +3516,6 @@ def private q51_gemv_gen(var gc : LlvmCodeCtx) : bool { var gexit = LLVMAppendBasicBlockInContext(gc.jit.ctx, gc.impl, "gexit") LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) let qgp = LLVMGetParam(gc.impl, 1u) @@ -3613,7 +3578,6 @@ def private kq_tile_gen_impl(var gc : LlvmCodeCtx; fmt : int) : bool { // entry: per-token activation/scale/bsum bases; the block unit is a superblock (nb = n/256) LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) te.wg = LLVMGetParam(gc.impl, 1u) @@ -3707,7 +3671,6 @@ def private q51_tile_gen(var gc : LlvmCodeCtx) : bool { // entry: per-token activation/scale/asum bases; the block unit is 32 weights (nb = n/32) LLVMPositionBuilderAtEnd(b, entry) - frame_align64(b, gc.jit.types) var sa = SliceArgs(gc_impl = gc.impl, ctx = gc.jit.ctx, kstep = 1) sa.yp = LLVMGetParam(gc.impl, 0u) te.wg = LLVMGetParam(gc.impl, 1u) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index c2f71d6043..5b4c8ef433 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -914,6 +914,7 @@ def q8_region_to_metal_blob(qp : int8 const?; sp : float const?; n, rows : int64 // Gather one PREPARED K-quant weight stack into the GPU tier's device planes — quant payloads // row-major per superblock in the k/k+16 nibble pairing, plus DECODED 20B scale rows. Reads // whatever the load prepared (grp interleave or disk-order); device bytes are identical either way. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice_rows : int64; repacked : bool; mr : int64; var wq : array; var ws : array) { let nsb = n / KQ_SUPERBLOCK_ELEMS @@ -968,7 +969,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(96l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.k3) { // qs columns 2blk + half carry a sub-block's four j sites at 2j; hmask column blk its eight sites at bit s + } elif (fmt == KqFmt.k3) { // the sub-block-packed qs + hmask columns for (m in range64(64l)) { let half = (m % 32l) / 16l let j = (m % 16l) / 4l @@ -1011,7 +1012,7 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (bj in range64(32l)) { dq[128l + bj] = gq[128l * mr + bj * mr + r] } - } elif (fmt == KqFmt.k6) { // qh columns 2blk + half carry one sub-block's four j sites at 2j; the disk byte h*32 + half*16 + j*4 + t carries sub-blocks 4h.. at 2b + } elif (fmt == KqFmt.k6) { // the sub-block-packed qh columns for (p in range64(64l)) { let half = (p % 32l) / 16l let j = (p % 16l) / 4l diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 4296d25035..5a2a72d430 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -500,9 +500,7 @@ def public get_matmul_min_chunk_rows_gemv() : int { return g_matmul_min_chunk_ro // GEMV tail: the one-chunk-per-lane equal split leaves the join waiting out the SLOWEST lane's // whole chunk (median lane idles 14-24% of every GEMV stage, per-lane trace 3990X). >1 publishes -// wave-aligned chunks per lane so finished lanes keep serving; default 16 (zen4 16 lanes, d=32768: iq4xs -// 3108 -> 2978 us, q51 4192 -> 4132 against 8, lanes balanced; the grain only bites past 64*waves*lanes rows); -// 1 = historical equal split. +// wave-aligned chunks per lane so finished lanes keep serving; 16 (only bites past 64*waves*lanes rows); 1 = equal split. var g_gemv_chunks_per_lane = 16 def public set_gemv_chunks_per_lane(v : int) { g_gemv_chunks_per_lane = max(v, 1) } def public get_gemv_chunks_per_lane() : int { return g_gemv_chunks_per_lane } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index 07be147cc4..a8188ea8df 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -1058,6 +1058,7 @@ def private dequant_iq4xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } //! k3 grp row dequant: the 2-bit lanes and hmask bits stay in place per (h, column) — w = (d * sc16) * (q - 4). +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def private dequant_k3_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { unsafe { for (sbi in range64(nsb)) { @@ -1291,6 +1292,7 @@ def private dequant_iq2xxs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr //! Dequant one row off the grp-REPACKED K-quant planes (repack_k4/k5/k6_grp layout). //! kqg/ksg = row's GROUP plane base (group g = row/mr); r = row % mr. //! embed_row's kq_repacked path — per-token, so scalar is fine. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr, n : int64; var dst : float?) { // nolint:STYLE037,STYLE038 — the flat per-format dispatch (one arm per kq format) is the honest shape let nsb = n / 256l let qsb = kq_qsb(int(fmt)) @@ -1367,7 +1369,7 @@ def dequant_kq_row_grp(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, m qlo |= ((hb >> int(t)) & 1) << 4 qhi |= ((hb >> int(4l + t)) & 1) << 4 } elif (fmt == 6l) { - let hbase = qb + 128l * mr + blk * 2l * mr * 4l // qh columns 2blk / 2blk + 1 hold this sub-block's four j sites at 2j + let hbase = qb + 128l * mr + blk * 2l * mr * 4l let h0 = uint(kqg[hbase + r * 4l + t]) let h1 = uint(kqg[hbase + (mr + r) * 4l + t]) qlo |= int(((h0 >> sj) & 3u) << 4u) diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index cbaf47f98e..8019736afe 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -162,6 +162,7 @@ def mx4q8_tile_gen(var yp : float?; wn : uint8 const?; we : uint8 const?; xqp : //! One row's dot off the grp K-quant planes, scalar — the kq stubs' reference body (runs //! only when the stamp declined) and the tests' repack oracle; bit-exact vs the portable disk //! dots. Activations are Q8_K-form: xsp per-superblock scale, xbsp per-16 quant sums. +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def kq_grp_row_dot(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { var acc = 0.0 let nsb = n / 256l @@ -188,7 +189,7 @@ def kq_grp_row_dot(fmt : int64; kqg : uint8 const?; ksg : uint8 const?; r, mr : qlo |= ((hb >> int(t)) & 1) << 4 qhi |= ((hb >> int(4l + t)) & 1) << 4 } elif (fmt == 6l) { - let hbase = qb + 128l * mr + blk * 2l * mr * 4l // qh columns 2blk / 2blk + 1 hold this sub-block's four j sites at 2j + let hbase = qb + 128l * mr + blk * 2l * mr * 4l let h0 = uint(kqg[hbase + r * 4l + t]) let h1 = uint(kqg[hbase + (mr + r) * 4l + t]) qlo |= int(((h0 >> sj) & 3u) << 4u) @@ -532,7 +533,7 @@ def q40q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k4q8_gemv_gen", gen = "dasllama_gemm_gen::k4_gemv"), tune_companion(fn = "k4q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k4_tile", @@ -553,7 +554,7 @@ def k4q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k5q8_gemv_gen", gen = "dasllama_gemm_gen::k5_gemv"), tune_companion(fn = "k5q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k5_tile", @@ -574,7 +575,7 @@ def k5q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k6q8_gemv_gen", gen = "dasllama_gemm_gen::k6_gemv"), tune_companion(fn = "k6q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k6_tile", @@ -595,7 +596,7 @@ def k6q8_tile_gen(var yp : float?; kqg : uint8 const?; ksg : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "q40q8_gemv_gen", gen = "dasllama_gemm_gen::q40_gemv"), tune_companion(fn = "q40q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q40_tile", @@ -628,7 +629,7 @@ def iq4xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq4xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq4xs_gemv"), tune_companion(fn = "iq4xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq4xs_tile", @@ -661,7 +662,7 @@ def k3q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k3q8_gemv_gen", gen = "dasllama_gemm_gen::k3_gemv"), tune_companion(fn = "k3q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k3_tile", @@ -694,8 +695,8 @@ def iq3sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq3sq8_gemv_gen", gen = "dasllama_gemm_gen::iq3s_gemv"), tune_companion(fn = "iq3sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3s_tile", @@ -771,8 +772,8 @@ def iq3xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq3xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq3xxs_gemv"), tune_companion(fn = "iq3xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq3xxs_tile", @@ -836,7 +837,7 @@ def iq4nlq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "iq4nlq8_gemv_gen", gen = "dasllama_gemm_gen::iq4nl_gemv"), tune_companion(fn = "iq4nlq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq4nl_tile", @@ -906,7 +907,7 @@ def k2q8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp : tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "k2q8_gemv_gen", gen = "dasllama_gemm_gen::k2_gemv"), tune_companion(fn = "k2q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::k2_tile", @@ -979,8 +980,8 @@ def iq2sq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq2sq8_gemv_gen", gen = "dasllama_gemm_gen::iq2s_gemv"), tune_companion(fn = "iq2sq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2s_tile", @@ -1052,8 +1053,8 @@ def iq2xsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; xq tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq2xsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xs_gemv"), tune_companion(fn = "iq2xsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xs_tile", @@ -1127,8 +1128,8 @@ def iq2xxsq8_gemv_gen(var yp : float?; kqp : uint8 const?; ksp : uint8 const?; x tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), // the gemv through the VBMI symbol lattice; the tile is the 512 row's + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, grid = "vbmi", requires = "avx512vnni,avx512bw,avx512vbmi"), tune_companion(fn = "iq2xxsq8_gemv_gen", gen = "dasllama_gemm_gen::iq2xxs_gemv"), tune_companion(fn = "iq2xxsq8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::iq2xxs_tile", @@ -1326,7 +1327,7 @@ def q51q8_gemv_gen(var yp : float?; qgp : uint8 const?; sgp : uint8 const?; xqp tune_perm(dot = "maddubs", width = 256, mr = 8, requires = "avx2"), tune_perm(dot = "vpdpbusd", width = 256, mr = 8, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), // the gemv's same-layout alternative to the 512 tile crown + tune_perm(dot = "vpdpbusd", width = 256, mr = 16, requires = "avx512vnni,avx512bw"), tune_companion(fn = "q51q8_gemv_gen", gen = "dasllama_gemm_gen::q51_gemv"), tune_companion(fn = "q51q8_layout_gen", gen = "dasllama_gemm_gen::q8q8_layout"), tune(gen = "dasllama_gemm_gen::q51_tile", diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index 9ed108f008..da740e1c32 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -56,7 +56,7 @@ def repack_q8q8_grp(var wp : int8?; var sp : float?; n, d, mr : int64; wbias : i let ncg = n / kgroup var tw : array var ts : array - tw |> reserve(d * n) // the plane-size scratch: reserve first, a bare resize past 64 MB trips max_unreserved_size + tw |> reserve(d * n) tw |> resize(d * n) ts |> reserve(d * nb) ts |> resize(d * nb) @@ -293,6 +293,7 @@ def repack_k5_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def repack_k6_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let nsb = n / 256l let qrow = nsb * 192l @@ -326,7 +327,7 @@ def repack_k6_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { } } } - for (blk in range64(8l)) { // qh re-packed per sub-block: column 2blk + half holds the four j sites' 2-bit fields at 2j + for (blk in range64(8l)) { // qh re-packed per sub-block for (half in range64(2l)) { for (t in range64(4l)) { var byte = 0u @@ -443,6 +444,7 @@ def repack_iq4xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { //! k3 grp layout: the qs bytes as 16 four-byte columns ([h 0..1][8 l-groups]) x mr rows, then the //! hmask as 8 columns x mr rows at 64*mr — the 2-bit lanes and mask bits stay in place (the //! kernels shift per block); scales interleave as k6's row ([16 x mr int8][mr x f16 d]). +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let nsb = n / 256l let qrow = nsb * 96l @@ -466,7 +468,7 @@ def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { for (r in range64(mr)) { let sq = (g * mr + r) * qrow + sbi * 96l let ss = (g * mr + r) * srow + sbi * 18l - for (blk in range64(8l)) { // qs: column 2blk + half holds the sub-block's four j sites at 2j; hmask: column blk, its eight sites at bit s (lo j, hi 4 + j) + for (blk in range64(8l)) { // qs and hmask re-packed per sub-block for (half in range64(2l)) { for (t in range64(4l)) { var qbyte = 0u @@ -994,6 +996,7 @@ def unpack_iq2xxs_panel_grp(kqg : uint8 const?; var dst : uint8?; mr, nsb : int6 } } +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, nsb : int64) { if (fmt == 33l) { unpack_iq3s_panel_grp(kqg, dst, mr, nsb) @@ -1025,7 +1028,7 @@ def unpack_kq_panel_grp(fmt : int64; kqg : uint8 const?; var dst : uint8?; mr, n let qb = sbi * (fmt == 5l ? 160l : 192l) * mr let ob = sbi * 256l * mr for (blk in range64(8l)) { - let hbase = qb + 128l * mr + blk * 2l * mr * 4l // k6 high plane: columns 2blk (lo half) and 2blk + 1 (hi); k5 ignores + let hbase = qb + 128l * mr + blk * 2l * mr * 4l // k6 high plane (k5 ignores) for (j in range64(4l)) { let sj = uint64(2l * j) let gcol = ((blk * 4l + j) * mr) * 4l diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index adaa9b5440..ee57704d2e 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -451,9 +451,8 @@ def kq_tile_entry(fmt : int64) : string => fmt == 40l ? "q40q8_tile_gen" : (fmt //! the gemv companion's own manifest entry: the tile's name with the tier swapped def kq_gemv_entry(fmt : int64) : string => replace(kq_tile_entry(fmt), "_tile_gen", "_gemv_gen") -//! the gemv's seat: among the rows sharing the tile winner's layout (mr), the two best by tile time, the one -//! with the best streamed-decode time takes the gemv - only by the margin over the tile winner's own gemv, -//! and the incumbent keeps a tie; "" when the tile winner keeps it (no separate entry, the companion follows) +//! the gemv's seat among the tile winner's same-layout rows; "" when the tile winner keeps it +[arch(at = "../ARCHITECTURE_MEASUREMENT.md#gemv-seat")] def private kq_gemv_seat(fmt : int64; winner : int; names : array; vmr : array; tile, gemv : array) : string { var cand <- [for (i in range(length(names))); i; where vmr[i] == vmr[winner] && tile[i] < 1.0e29lf] cand |> sort() $(a, b) => tile[a] < tile[b] @@ -1318,6 +1317,7 @@ def kq_test_family(fmt : int64; kfxs : array) : bool { // and the gemv (streamed + hot decode shapes), print the table, and return the tile-best // suffix ("" = a gate failed). The kq gemv is nrsplit-independent — same-mr rows share the // plane and the gemv shape — so the tile bench decides the family entry. +[arch(at = "../ARCHITECTURE_MEASUREMENT.md#gemv-seat")] def kq_tune_family(fmt : int64; var rows_out : table; var gemv_out : string&) : string { // nolint:STYLE037,STYLE038 — one family race: shared repacked buffers couple gate, bench and the two picks // batch shape: fat STREAMING ffn (weights > the L2 budget, multi-token-block walk) — the shape // class that carries ~90% of prefill time. The old 2048x512x64 kv-projection probe was L2-hot @@ -1444,8 +1444,6 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge } var tnames <- [for (t in tvs); t._0] winner = gen_sticky(kq_tile_entry(fmt), tnames, best, winner) - // the seat is decided at the engine's decode shape - a DRAM-bound plane (the streamed fixture fits L3 on - // server parts) streamed by every lane through the engine's splitter; the one-lane box keeps the streamed time var tbest : array tbest |> resize(nv) for (i in range(nv)) { @@ -1453,10 +1451,6 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge } with_job_que() { // the lane count and the team dispatch both need the live jobque if (get_total_hw_jobs() > 1) { - // the engine's row length matters (k3 on Granite Rapids: 256_mr16 wins at n=2048, loses at 14336), so the - // seat fixture is a short one at the ffn width, tiled: 512 rows transcode in a blink; 320 tiles make 163840 - // rows - 600 MB for the 2-bit grids, 1.9 GB for k6 - past the 504 MB L3 a Granite Rapids socket lends a - // slice (a 300 MB plane came back L3-resident in some rounds and the seat went to the cached-mode winner) var kefx <- build_kq_fixture(fmt, 14336l, 512l, 1l) let TILES = 320l var ktfx = KqFixture(fmt = kefx.fmt, n = kefx.n, d = kefx.d * TILES, ntok = 1l, nsb = kefx.nsb) @@ -1488,7 +1482,7 @@ def kq_tune_family(fmt : int64; var rows_out : table; var ge } } for (vi in range(nv)) { - if (length(tsamples[vi]) > 0) { + if (!empty(tsamples[vi])) { sort(tsamples[vi]) tbest[vi] = tsamples[vi][length(tsamples[vi]) / 2] } @@ -2218,12 +2212,10 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family print("aborting mid-run - no sidecar written\n") return false } - // the kq families tune separately — the tile-best row is the format's entry, and the gemv gets its - // own entry when a same-layout row beats the tile seat on the streamed decode shape (the layout - // companion pins the plane mr, so only same-mr rows can differ; the two best by tile time race). No - // child-process confirm gate here: the kq stamp only moves the kq planes' interleave, - // and the e2e exposure (a tile-optimal mr shaving decode kernel rate) is bounded by the - // decode path staying DRAM-bound — validated end-to-end when the entries first landed. + // the kq families tune separately — the tile-best row is the format's entry, and the gemv takes its + // own entry when its seat differs. No child-process confirm gate here: the kq stamp only moves the + // kq planes' interleave, and the e2e exposure (a tile-optimal mr shaving decode kernel rate) is + // bounded by the decode path staying DRAM-bound — validated end-to-end when the entries first landed. for (fmt in kqSel) { interrupt_gate("k{fmt}q8_tile_gen") let kq_t0 = ref_time_ticks() diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh index 1268239cc8..47f135ed3b 100644 --- a/modules/dasLLAMA/harness/kernel_ladder.sh +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -3,8 +3,8 @@ # one thread, normal tune mode = the winner this box ships) against the reference exe's # test-backend-ops perf at the same shape, decode (n=1) and prefill (n=512) rows, joined with # the ratio (reference us / ours us; >= 1.0 = ours is faster). No model, no jobque: the kernel -# and nothing else. The bench's arena sits page-aligned (--base-align 4096) - the engine's image -# planes start on 16 KiB boundaries, and k6/k3 read 10-30% differently at the heap's random phase. The reference binary needs the GGML_BENCH_THREADS define (harness README). +# and nothing else. The bench's arena sits page-aligned (--base-align 4096), the phase the engine's +# image planes give the kernels. The reference binary needs the GGML_BENCH_THREADS define (harness README). # # Usage: kernel_ladder.sh [fmt,fmt,...] (default: every format) # KL_DASLANG the daslang binary (default: this tree's bin/daslang, or bin/Release/daslang.exe on From b94f2ef082f4718019842500fdab3251dd503d8e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 17:38:50 -0700 Subject: [PATCH 110/123] tune profile arm-i8mm: re-minted on the rebased branch (M4 Pro; 49 entries, TEST 103 ok); the M4 v2 table --- .../defaults/arm-i8mm.tune-defaults.json | 28 +++++++++---------- plans/kernel_parity_pass.md | 24 ++++++++++++++++ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json b/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json index c0cf3da8a0..3c3020b4f5 100644 --- a/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json +++ b/modules/dasLLAMA/performance/defaults/arm-i8mm.tune-defaults.json @@ -1,24 +1,24 @@ { "kernels" : { - "add_inplace" : "vec16_u2", - "cvt_f32_to_f16" : "vec16", + "add_inplace" : "vec8_u2", + "cvt_f32_to_f16" : "vec8_u2", "rope_scaled_neox_tab" : "vec8_u2", - "softmax" : "vec8_u2", "q51q8_tile_gen" : "mr8", + "softmax" : "vec8_u2", "mul_inplace" : "vec8_u2", "quantize_q8_0_bs_into_ptr" : "plain", - "dot_q8kv" : "vec4_u4", "iq2xsq8_tile_gen" : "mr8", + "dot_q8kv" : "vec4_u4", "dot_q8q8" : "vec16", - "quantize_q8kv_row" : "plain", "iq4xsq8_tile_gen" : "mr8", "iq4nlq8_tile_gen" : "mr8", + "quantize_q8kv_row" : "plain", "axpy_f16" : "vec8_u2", "q40q8_tile_gen" : "mr8", "axpy_tq4kv" : "vec8_u2", "cvt_tq4kv_to_f32" : "vec8_u2", "axpy" : "vec8_u2", - "dot_q8q8kv" : "plain", + "dot_q8q8kv" : "vec16", "dot_mx4q8" : "u2", "softmax_sink" : "vec8_u2", "iq2xxsq8_tile_gen" : "mr8", @@ -29,15 +29,15 @@ "cvt_q8kv_to_f32" : "vec8_u2", "axpy_q8kv" : "vec8_u2", "dot_q8q8_f16s" : "vec16", - "q8q8_tile_gen" : "mr8_budget", + "q8q8_tile_gen" : "kstep2", "quantize_q8_0_into_ptr" : "plain", "k4q8_tile_gen" : "mr8", - "gemm_f32_uk_4x16" : "u2", - "dot_q51e" : "vec16", "k5q8_tile_gen" : "mr8", "k6q8_tile_gen" : "mr8", "k3q8_tile_gen" : "mr8", "k2q8_tile_gen" : "mr8", + "gemm_f32_uk_4x16" : "u2", + "dot_q51e" : "vec16", "dot_f16" : "vec8_u2", "cvt_f16_to_f32" : "vec8_u2", "dot" : "vec8_u2", @@ -48,22 +48,22 @@ "dot_q4" : "vec4_u4", "quantize_tq4kv_row" : "plain", "copy_floats" : "vec8_u2", - "rmsnorm" : "vec8" + "rmsnorm" : "vec8_u2" }, "provenance" : { "validation" : "ok", - "noise_probes" : "start cv 0.36%; mid1 cv 0.38%; mid2 cv 0.48%; end cv 0.27%", + "noise_probes" : "start cv 1.24%; mid1 cv 1.35%; mid2 cv 1.70%; end cv 1.43%", "platform" : "darwin", - "noise_floor_cv_pct" : "0.48", + "noise_floor_cv_pct" : "1.70", "features" : "neon;dotprod;i8mm;bf16;fullfp16;lse", "class" : "arm-i8mm", - "written" : "2026-09-01T15:43:02.269Z", + "written" : "2026-09-02T00:17:51.676Z", "validation_demoted" : "0", "mode" : "normal", "dasllama_version" : "15", "origin" : "profile", "noise" : "ok", "arch" : "arm64", - "validation_max_drift_pct" : "1.09" + "validation_max_drift_pct" : "0.27" } } \ No newline at end of file diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 9c9bc2e55e..204338fbbb 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -569,6 +569,30 @@ set (ruled, followup_general 67) - iq4xs 0.91-0.94, k5 0.93-0.98, iq4nl 0.94-0.9 byte-capped at 0.944 (q8s16, the GGUF q8_0 path, reads 0.96). The grid families' tile crowns are the vbmi rows (a tile tie - the lattice row's tile IS the 512 body - so the gemv companion carries the lattice without a separate seat). +### M4 Pro v2 - arm-i8mm re-minted on the rebased branch (2026-09-01; SOLO one thread, TEAM 10 perf-core lanes; ratio = reference / ours) + +| fmt | gemv 1T | tile 1T | gemv 10L | +|---|---|---|---| +| q8 | 1.65 | 2.31 | 0.95 | +| q8s16 | 1.62 | 2.24 | 1.02 | +| k4 | 1.40 | 2.18 | 1.00 | +| k5 | 1.10 | 3.80 | 1.09 | +| k6 | 1.34 | 2.22 | 1.11 | +| q40 | 2.08 | 2.37 | 1.24 | +| q51 | 1.24 | 2.91 | 1.56 | +| iq4xs | 1.68 | 3.22 | 1.14 | +| k3 | 1.27 | 2.60 | 1.46 | +| iq3s | 1.04 | 13.51 | 1.16 | +| iq3xxs | 1.09 | 12.41 | 1.18 | +| iq4nl | 1.85 | 3.18 | 1.30 | +| k2 | 1.44 | 2.54 | 1.52 | +| iq2s | 1.24 | 8.17 | 1.13 | +| iq2xs | 0.90 | 6.79 | 0.94 | +| iq2xxs | 1.17 | 8.58 | 1.22 | +| mx4 | 1.35 | 2.59 | 1.51 | + +Every row at or past 0.95 except iq2xs (0.90 one thread, 0.94 at 10 lanes) - the arm-i8mm class declines the lattice (VBMI is x86), so iq2xs rides the sdot row form there; the v1 table read it past 0.96 - a re-measure is owed before the PR. + ## 3. Research memos (read before touching the kernels) - `kernel_parity_research_cpu.md` - llama.cpp's CPU vec_dot for the five grid formats + Q2_K, From 44cbfb2e0cbf065a2169aa8c72e2168ee89b0d8c Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 17:43:42 -0700 Subject: [PATCH 111/123] followup_general 68: iq2xs on arm-i8mm regressed with the shared row-group decode (0.90/0.95; the ARM panel/row choice is the fix) --- modules/dasLLAMA/followup_general.md | 12 ++++++++++++ plans/kernel_parity_pass.md | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index a15c6d8cc0..406541341e 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -856,3 +856,15 @@ applies the format's cap (its callers know the tensor's KqFmt). Ruled 2026-09-01: zen4's iq4xs 0.92 / iq4nl 0.94 / k5 0.93-0.97 are accepted as they stand; pinned affinity is normally the faster arrangement and this may be the one case it is not - decide with a second SMT box in hand. + +68. **iq2xs on arm-i8mm regressed with the shared row-group grid decode.** M4 Pro, engine shape + (10 perf-core lanes, d=32768): iq2xs 1530 us at the arc's mid-point (branch 8693a5b47) -> 2348 + now (0.95 of the reference; 0.90 at one thread). The seat is unchanged (`mr8`, the NEON sdot + row form) and the chunk grain does not move it (8/16/4/1 all ~2350). What changed is the grid + decode: on ARM `grid_rows_path` returns the ROW-GROUP form for every grid format (`rv == 4`), + and the refactor that made row groups win on x86 (0.79 -> 1.6 for iq2xs there) cost the ARM + path, which the panel form had served better for iq2xs's 9-bit index. There is no knob to force + the panel form on ARM (`DASLLAMA_GRID_ROWS_X86` gates x86 only). Isolated to arm-i8mm iq2xs - + every other M4 row is >= 0.95 and the goal boxes (zen4, Intel) are unaffected. The fix is an ARM + panel/row-form choice for iq2xs, decided the way the x86 class gate is: measure both forms on M4 + and pin the winner per format. diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index 204338fbbb..c6612b8685 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -591,7 +591,7 @@ the lattice row's tile IS the 512 body - so the gemv companion carries the latti | iq2xxs | 1.17 | 8.58 | 1.22 | | mx4 | 1.35 | 2.59 | 1.51 | -Every row at or past 0.95 except iq2xs (0.90 one thread, 0.94 at 10 lanes) - the arm-i8mm class declines the lattice (VBMI is x86), so iq2xs rides the sdot row form there; the v1 table read it past 0.96 - a re-measure is owed before the PR. +Every row at or past 0.95 except iq2xs (0.90 one thread, 0.95 at 10 lanes) - a real regression from v1's 1.46, isolated to the ARM sdot row form (followup_general.md entry 68: the shared row-group refactor that won x86 cost ARM iq2xs; the fix is an ARM per-format panel/row choice). Chunk grain measured a null (8/16/4/1 all ~2350 us). ## 3. Research memos (read before touching the kernels) From a467ee512b7b38eff8e5184ee905f256e3ff00c1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 17:50:22 -0700 Subject: [PATCH 112/123] followup_general 68 corrected: M4 iq2xs 0.90/0.95 is the format's true ARM standing - the old 1.46 does not reproduce with its own sources; the row form beats the panel on ARM everywhere --- modules/dasLLAMA/followup_general.md | 19 ++++++++----------- plans/kernel_parity_pass.md | 2 +- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 406541341e..a174e241b8 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -857,14 +857,11 @@ 0.92 / iq4nl 0.94 / k5 0.93-0.97 are accepted as they stand; pinned affinity is normally the faster arrangement and this may be the one case it is not - decide with a second SMT box in hand. -68. **iq2xs on arm-i8mm regressed with the shared row-group grid decode.** M4 Pro, engine shape - (10 perf-core lanes, d=32768): iq2xs 1530 us at the arc's mid-point (branch 8693a5b47) -> 2348 - now (0.95 of the reference; 0.90 at one thread). The seat is unchanged (`mr8`, the NEON sdot - row form) and the chunk grain does not move it (8/16/4/1 all ~2350). What changed is the grid - decode: on ARM `grid_rows_path` returns the ROW-GROUP form for every grid format (`rv == 4`), - and the refactor that made row groups win on x86 (0.79 -> 1.6 for iq2xs there) cost the ARM - path, which the panel form had served better for iq2xs's 9-bit index. There is no knob to force - the panel form on ARM (`DASLLAMA_GRID_ROWS_X86` gates x86 only). Isolated to arm-i8mm iq2xs - - every other M4 row is >= 0.95 and the goal boxes (zen4, Intel) are unaffected. The fix is an ARM - panel/row-form choice for iq2xs, decided the way the x86 class gate is: measure both forms on M4 - and pin the winner per format. +68. **iq2xs on arm-i8mm reads 0.90 one-thread / 0.95 at the engine shape - the format's true ARM standing, + not a regression.** An earlier mid-arc table read 1.46 (1530 us at 10 lanes), but neither today's emitter + nor that commit's own `dasllama/` sources reproduce it on the same box (both measure ~2350; binary, bench + and reference identical) - the old figure was an artifact of that session. The decode form is settled: the + row-group form beats the panel on ARM for every grid format at both shapes (iq2xs panel 3885 vs rows 2365 + at 10 lanes), so the sdot gate is already right. What would actually lift iq2xs on ARM is a cheaper 9-bit + index path in the row form (its u16 word costs two column-byte reads per site where iq2xxs reads one), or + an ARM analog of the x86 symbol lattice over SMMLA - both open kernel work, not a gate flip. diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index c6612b8685..c14a96a7e8 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -591,7 +591,7 @@ the lattice row's tile IS the 512 body - so the gemv companion carries the latti | iq2xxs | 1.17 | 8.58 | 1.22 | | mx4 | 1.35 | 2.59 | 1.51 | -Every row at or past 0.95 except iq2xs (0.90 one thread, 0.95 at 10 lanes) - a real regression from v1's 1.46, isolated to the ARM sdot row form (followup_general.md entry 68: the shared row-group refactor that won x86 cost ARM iq2xs; the fix is an ARM per-format panel/row choice). Chunk grain measured a null (8/16/4/1 all ~2350 us). +Every row at or past 0.95 except iq2xs one-thread (0.90; 0.95 at the engine shape) - NOT a regression: v1's 1.46 does not reproduce with v1's own sources on the same box (both ~2350 us), and the row form beats the panel on ARM for every grid at both shapes (iq2xs panel 3885 vs rows 2365 at 10 lanes). followup_general.md entry 68 carries the true standing and the two kernel-work directions; chunk grain is a null (8/16/4/1 all ~2350). ## 3. Research memos (read before touching the kernels) From e20600b1b92ef3f7e7b86b9c083b28a5f559c533 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 18:05:39 -0700 Subject: [PATCH 113/123] dasLLVM review-audit fixes: the tier-gates section's two factual bugs, the GEP mechanism as its own anchored section, the tier rule as the invariant that holds, the emitter pin recomputed Eight gates, not nine; a generic target drops to forced-only truth rather than all-off. ARCHITECTURE.md#gep-constant-fold carries the ConstantExpr mechanism, cited from the llvm_boost wrapper; the REVIEW rule shrinks to the decidable ban (never LLVMSetIsInBounds). The three-part tier rule becomes "a name used in requires= or a gate has its cpuid line in the same diff" - five known names legitimately have no gate of their own; #x64-tier-gates gains the fingerprint citer. LLVM_JIT_EMITTER_HASH recomputed for the llvm_boost change (0x26d3a061cfe6fc27). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLVM/ARCHITECTURE.md | 24 ++++++++++++++++++------ modules/dasLLVM/REVIEW.md | 21 +++++++++++---------- modules/dasLLVM/daslib/llvm_boost.das | 1 + modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- modules/dasLLVM/daslib/llvm_tune.das | 1 + 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md index 41552bd0c0..cae1d0740b 100644 --- a/modules/dasLLVM/ARCHITECTURE.md +++ b/modules/dasLLVM/ARCHITECTURE.md @@ -164,19 +164,31 @@ the manifest's runtime readers reach the same call unconditionally. ## 6. The x64 kernel-matrix tier gates {#x64-tier-gates} -An x64 host target publishes nine boolean gates (`g_target_x64_*`), one per instruction tier the +An x64 host target publishes eight boolean gates (`g_target_x64_*`), one per instruction tier the kernel matrix and the tune grids select on: `avx2`, `f16c`, `vnni256` (256-bit VPDPBUSD by either VEX AVX-VNNI or EVEX AVX512-VNNI+VL), `avx512bw` (zmm byte ops - BW, not merely F), `avx512vnni` (zmm VPDPBUSD; implies bw, the sign trick around it is BW), `avx512vbmi` (VPERMI2B / VPERMB / VPMULTISHIFTQB - the grid formats' symbol lattice), `vnniint8` (VEX VPDPBSSD, native s8 x s8), and `amx` (both amx-tile and amx-int8; the per-process XTILEDATA grant is a separate runtime step the family's own witness performs). `init_jit_target_flags` decides each from cpuid truth OR'd with -the `DAS_JIT_X64_FORCE_FEATURES` emission-only override, and forced-only on a cross triple; a -generic target (`host_features = false`) leaves every gate off. +the `DAS_JIT_X64_FORCE_FEATURES` emission-only override; a cross triple or a generic target +(`host_features = false`) drops to forced-only truth - cpuid is not consulted, the force env is +the only tier source there. The cpuid truth is `das_cpu_supports` (`src/builtin/module_builtin_runtime.cpp`), a hand-kept table keyed by the LLVM target-feature spelling - so the force env and `llc -mattr` take the same names. -A tier flag therefore has three parts that must land together: its cpuid line there, its name in +A tier feature usually lands as three parts: its cpuid line there, its name in `TUNE_KNOWN_FEATURES` (`daslib/llvm_tune.das`, the profile fingerprint the `requires=` gates are -checked against), and the `g_target_x64_*` gate. A name missing from the table answers false on -every box - the perm that requires it declines everywhere and no error names the cause. +checked against), and - when the emitters branch on it - a `g_target_x64_*` gate. The cpuid line +is the load-bearing one: a name missing from the table answers false on every box, so the perm +that requires it declines everywhere and no error names the cause. + +## 7. A constant-folded GEP is not an instruction {#gep-constant-fold} + +`LLVMBuildGEP2` over a global with a constant index does not create an instruction - LLVM folds +it into a `ConstantExpr`, one shared object per distinct expression in the context. Any API that +casts a "just built" GEP to `GetElementPtrInst` therefore writes through the wrong type into the +constant's memory when the fold happened - `LLVMSetIsInBounds` was the instance that corrupted +the context (heap damage surfacing in `LLVMContextDispose` at teardown). The in-bounds form is +requested at build time (`LLVMBuildInBoundsGEP2`), which folds to an in-bounds `ConstantExpr` +correctly; the `llvm_boost` wrapper's `inbounds` default rides that builder. diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index a0e32420a4..0c4deff6cf 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -71,13 +71,14 @@ recorded path is read by other people and must not name the user who minted it. A host path a diff passes to a filesystem call stays raw: no filesystem call resolves `~`. `tests/llvm_tune_manifest.das` here asserts a minted sidecar carries no home directory. -- **Never set the in-bounds flag on a GEP after building it - `LLVMSetIsInBounds` casts its - argument to `GetElementPtrInst`; pass `inbounds` to `LLVMBuildGEP2` in `daslib/llvm_boost.das`, - or call `LLVMBuildInBoundsGEP2`, instead.** A GEP over a global with a constant index folds - into a `ConstantExpr`, so that cast writes the flag through a wrong type into LLVM's constant - object. -- **A new x64 tier feature lands in three places in one diff: its cpuid line in `das_cpu_supports` - (`src/builtin/module_builtin_runtime.cpp`), its name in `TUNE_KNOWN_FEATURES` - (`daslib/llvm_tune.das`), and its `g_target_x64_*` gate in `daslib/llvm_jit_common.das` - (`ARCHITECTURE.md#x64-tier-gates`).** A name the cpuid table does not know answers false on - every box, so every perm that requires it silently declines to its fallback. + +- **Never call `LLVMSetIsInBounds` - build the GEP in-bounds with `LLVMBuildInBoundsGEP2` (the + `llvm_boost` wrapper's `inbounds` default) instead.** A constant-folded GEP is a + `ConstantExpr`, and the setter's cast writes through the wrong type into it + (`ARCHITECTURE.md#gep-constant-fold`). + +- **A feature name used in a `requires=` list or a `g_target_x64_*` gate has its cpuid line in + `das_cpu_supports` (`src/builtin/module_builtin_runtime.cpp`) in the same diff** + (`ARCHITECTURE.md#x64-tier-gates`). A name the cpuid table does not know answers false on + every box, so every perm that requires it silently declines to its fallback and no error names + the cause. diff --git a/modules/dasLLVM/daslib/llvm_boost.das b/modules/dasLLVM/daslib/llvm_boost.das index 7de9ff92fd..a9f98d4d36 100644 --- a/modules/dasLLVM/daslib/llvm_boost.das +++ b/modules/dasLLVM/daslib/llvm_boost.das @@ -307,6 +307,7 @@ def LLVMAddIncoming(phi : LLVMOpaqueValue?; var phi_vals : array Date: Tue, 1 Sep 2026 18:08:43 -0700 Subject: [PATCH 114/123] dasLLAMA review-audit fixes: the bare model-scaled resize, the grain default reverted to 8, two stale layout comments, provenance on ledgers 67/68 and sec.2.26, lcpp provenance on the two bring-up lines, the reserve rule sharpened, the board re-mint duty as its own rule, the tests/REVIEW.md routing said as routing The 8 -> 16 grain change is dropped: its evidence never ran both variants interleaved in one process and the later sweeps on zen4 and M4 read the difference as noise - the ruled-accepted rows do not depend on it. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md | 2 +- modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md | 4 ++-- modules/dasLLAMA/REVIEW.md | 14 +++++++------- modules/dasLLAMA/REVIEW_MEASUREMENT.md | 5 +++++ modules/dasLLAMA/dasllama/dasllama_math.das | 4 ++-- .../dasLLAMA/dasllama/dasllama_math_default.das | 1 - modules/dasLLAMA/dasllama/dasllama_math_gen.das | 1 + modules/dasLLAMA/dasllama/dasllama_repack.das | 3 --- modules/dasLLAMA/followup_general.md | 8 +++++--- 9 files changed, 23 insertions(+), 19 deletions(-) diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index 9ac70a97f2..f2465a8b33 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -170,7 +170,7 @@ by the margin over the tile winner's own gemv, and the incumbent keeps a tie. Ev therefore carries a 256-wide `mr = 16` alternate beside its 512-wide tile crown. The seat is decided at the engine's decode shape - a DRAM-bound plane streamed by every lane through the engine's own splitter - because the engine's row length moves the answer (k3 on Granite Rapids: the 256 seat wins -at n=2048 and loses at 14336). The seat fixture is a 512-row build at the ffn width tiled 320 times, +at n=2048 and loses at 14336 - `benchmarks/matmul/kq_kernel_bench.das`, tune mode, seats pinned, d=32768). The seat fixture is a 512-row build at the ffn width tiled 320 times, past the largest L3 a socket lends a slice of, and the seat takes the MEDIAN of seven rounds: a round that finds the plane in L3 must not crown it. In normal mode `llvm_tune` stamps a companion from its own manifest entry when one exists and is a perm this box can run, else from the tile's. diff --git a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md index a5f5ae2514..bff6a3b94f 100644 --- a/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md +++ b/modules/dasLLAMA/HOW_TO_ADD_A_FORMAT.md @@ -567,7 +567,7 @@ only the step-3 0.0267 top-2 flip vs llama.cpp. M1 benches: CPU das 897.4/56.3 v is CLOSED - and with it THE FORMAT LADDER: four-tier table zen2 2.86x/0.70x, vk 0.78x/0.70x, M1 CPU 6.41x/0.57x, Metal 0.93x/0.78x. -2026-09-01 (kernel parity pass): zen2 16t vs clean-cpu on the IQ2_XXS-local 1B: pp512 506.9 vs +2026-09-01 (kernel parity pass): zen2 16t (`benchmarks/lcpp_bench.das --for-debug-purposes`, debug-jit) vs the clean-cpu `llama-bench` on the IQ2_XXS-local 1B: pp512 506.9 vs 181.2 (2.80x), tg128 88.6 vs 86.2 (1.03x, was 0.70x) - the sign column + u64 pair gemv. ### IQ2_XS Phase A (CPU, 2026-08-31) - the ksigns u64 tier @@ -637,7 +637,7 @@ tiers. M1 16GB benches: CPU das 746.6/51.6 vs llama.cpp 144.8/101.1 (5.16x/0.51x (0.93x/0.86x - the iq2s pp class). The format is CLOSED on all four tiers; four-tier table: zen2 2.78x/0.70x, vk 0.77x/0.54x, M1 CPU 5.16x/0.51x, Metal 0.93x/0.86x. -2026-09-01 (kernel parity pass): zen2 16t vs clean-cpu on the IQ2_XS-local 1B: pp512 480.7 vs +2026-09-01 (kernel parity pass): zen2 16t (`benchmarks/lcpp_bench.das --for-debug-purposes`, debug-jit) vs the clean-cpu `llama-bench` on the IQ2_XS-local 1B: pp512 480.7 vs 174.6 (2.75x), tg128 93.8 vs 85.9 (1.09x, was 0.70x) - sign column + u64 pair + column dword read. ### IQ2_S Phase A (CPU, 2026-08-31) - the u64-grid tier diff --git a/modules/dasLLAMA/REVIEW.md b/modules/dasLLAMA/REVIEW.md index 9d0bec2b5e..ae6bf27de4 100644 --- a/modules/dasLLAMA/REVIEW.md +++ b/modules/dasLLAMA/REVIEW.md @@ -27,8 +27,8 @@ winners back. **A change to the sidecar-exchange client (`dasllama/dasllama_exchange.das`), or to a tune-boot path that reaches it, applies `REVIEW_EXCHANGE.md`.** -**Every `dasllama/` change applies this folder's `tests/REVIEW.md`.** A `dasllama/` diff never -opens that checklist on its own. +**Every `dasllama/` change applies this folder's `tests/REVIEW.md` - open it explicitly: the +folder walk does not surface it for a `dasllama/`-only diff.** **A GPU kernel, driver, dispatch-class, or K/V-mirror change - and a GPU kernel A/B race, knockout, or hand-binding arm, wherever the diff puts it - applies `REVIEW_GPU.md`.** @@ -249,8 +249,8 @@ the tuning the profile was meant to save. root) - is a `def` returning it, never a module global with a declaration initializer (`let` or `var`).** A team lane never runs global initializers, so the global reads zero there while every single-threaded run reads the right value. -- **A `resize` of a buffer whose element count scales with a model dimension in `dasllama/` is - preceded by a `reserve` of the same count** (`reserve_resize` / `grow_resize` / - `ensure_length` in `dasllama_common.das`, or the pair spelled out). A bare grow past 64 MB trips - the heap's unreserved-size cap and panics the load; `dasllama_repack.das`'s plane-size scratch - copies were the last bare ones. +**A `resize` in `dasllama/` of a buffer whose element count scales with a model dimension is +preceded by a `reserve` of the same count (`reserve_resize` / `grow_resize` / `ensure_length` in +`dasllama_common.das`, or the pair spelled out) - whatever the size looks like at today's +shapes.** A model dimension makes the count unbounded, and a bare grow past the heap's +unreserved-size cap (64 MB) panics the load on the first big model rather than at the call site. diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 64d77f451b..c8cec4edf1 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -37,6 +37,11 @@ family, or backend no existing cell exercises - a q8 or f32 serving lane and a G included. A kernel or form that only makes a path an existing cell already serves faster is not a new capability - it re-mints that cell's row on at least one box instead. +**A diff that makes an already-served path measurably faster re-mints that cell's board row +(`performance/records/`) on at least one box, in the same change.** The board is the module's +public memory of what serving costs; a kernel win that never lands there is invisible to the +next regression check. + **A timing figure PRESENTED AS A MEASUREMENT of a served turn as a whole - tok/s, latency, a whole-turn model or engine comparison - is a defect wherever this module writes it down with no cell behind it: a checked-in doc, a ledger, a code comment, or a PR description.** The diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 5a2a72d430..868c319512 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -500,8 +500,8 @@ def public get_matmul_min_chunk_rows_gemv() : int { return g_matmul_min_chunk_ro // GEMV tail: the one-chunk-per-lane equal split leaves the join waiting out the SLOWEST lane's // whole chunk (median lane idles 14-24% of every GEMV stage, per-lane trace 3990X). >1 publishes -// wave-aligned chunks per lane so finished lanes keep serving; 16 (only bites past 64*waves*lanes rows); 1 = equal split. -var g_gemv_chunks_per_lane = 16 +// wave-aligned chunks per lane so finished lanes keep serving; default 8; 1 = historical equal split. +var g_gemv_chunks_per_lane = 8 def public set_gemv_chunks_per_lane(v : int) { g_gemv_chunks_per_lane = max(v, 1) } def public get_gemv_chunks_per_lane() : int { return g_gemv_chunks_per_lane } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index a8188ea8df..df27e5c492 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -1057,7 +1057,6 @@ def private dequant_iq4xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } } -//! k3 grp row dequant: the 2-bit lanes and hmask bits stay in place per (h, column) — w = (d * sc16) * (q - 4). [arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def private dequant_k3_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { unsafe { diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 8019736afe..16d0ac572f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -2093,6 +2093,7 @@ def private kq_batch_cell_gen(fmt : int; var myp : float?; kqp : uint8 const?; k let packed = kq_reads_packed_planes(int64(fmt)) var scratch : array if (!packed) { + scratch |> reserve(mr * nsb * 256l + 64l) scratch |> resize(mr * nsb * 256l + 64l) // + a cache line: the panel starts 64-aligned, the tile's 32/64-byte loads never straddle one } unsafe { diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index da740e1c32..c3fc143ecb 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -441,9 +441,6 @@ def repack_iq4xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } -//! k3 grp layout: the qs bytes as 16 four-byte columns ([h 0..1][8 l-groups]) x mr rows, then the -//! hmask as 8 columns x mr rows at 64*mr — the 2-bit lanes and mask bits stay in place (the -//! kernels shift per block); scales interleave as k6's row ([16 x mr int8][mr x f16 d]). [arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let nsb = n / 256l diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index a174e241b8..293fb9aed3 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -844,8 +844,9 @@ the darwin branch then applies unchanged. No EC2 instance has such a part (server Xeons only); a Hetzner EX44 (i5-13500, 6P+8E) or EX101 (i9-13900) rents by the hour for the check. -67. **Per-format decode lane cap on SMT x86.** zen4 (8 cores x 2 SMT) at the engine's decode shape - (d=32768, DRAM-streamed): the light formats gain at one lane per core - iq4xs 3116 -> 3004 us, +67. **Per-format decode lane cap on SMT x86.** Figures: `benchmarks/matmul/kq_kernel_bench.das` + (`--team --d 32768 --ntok 0 --base-align 4096`, DAS_JOBQUE_THREADS as stated) against `test-backend-ops perf` + at m=32768/16t on the same box. zen4 (8 cores x 2 SMT) at the engine's decode shape (d=32768, DRAM-streamed): the light formats gain at one lane per core - iq4xs 3116 -> 3004 us, iq4nl 3049 -> 2946, q8s16 6339 -> 6120, k4 3169 -> 2891 - while the lattice grids halve without the SMT lanes (iq2xs 1412 -> 2714) and k5 does not move. Two siblings split one core's issue width on a kernel that is not ALU-bound; the reference streams the same bytes at 84-91 GB/s @@ -858,7 +859,8 @@ faster arrangement and this may be the one case it is not - decide with a second SMT box in hand. 68. **iq2xs on arm-i8mm reads 0.90 one-thread / 0.95 at the engine shape - the format's true ARM standing, - not a regression.** An earlier mid-arc table read 1.46 (1530 us at 10 lanes), but neither today's emitter + not a regression.** Figures: `benchmarks/matmul/kq_kernel_bench.das` (1T default shape; `--team --d 32768` + at 10 lanes) against `test-backend-ops perf` on the same M4 Pro. An earlier mid-arc table read 1.46 (1530 us at 10 lanes), but neither today's emitter nor that commit's own `dasllama/` sources reproduce it on the same box (both measure ~2350; binary, bench and reference identical) - the old figure was an artifact of that session. The decode form is settled: the row-group form beats the panel on ARM for every grid format at both shapes (iq2xs panel 3885 vs rows 2365 From 7b951e30fae0352709b37988515f259c830ce72f Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 18:14:39 -0700 Subject: [PATCH 115/123] style-hygiene round: dead gather names, family docs onto their owners, rowForm/uses_grid_rows/uses_sign_column/accLo-Hi/teamUs renames, the prefetch knob spelled out, PF_* constants, qhbyte, SOLO_BURN, ladder header - 25 findings applied, 4 declined consciously --- .../benchmarks/matmul/kq_kernel_bench.das | 3 ++- modules/dasLLAMA/dasllama/dasllama_layout.das | 16 ++++++++-------- modules/dasLLAMA/dasllama/dasllama_math.das | 2 +- .../dasLLAMA/dasllama/dasllama_math_default.das | 1 + modules/dasLLAMA/dasllama/dasllama_math_gen.das | 2 +- modules/dasLLAMA/dasllama/dasllama_repack.das | 7 ++++--- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 0e179fdf43..8d0df0fe00 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -260,6 +260,7 @@ struct RowStat { } let TEAM_BURN = 6 // warmup dispatches per row before the team arm times anything +let SOLO_BURN = 3 // the solo arm's unmeasured warm rounds (a core ramps for ~3) var g_each_round = false // --each //! the engine's decode dispatch, verbatim (dasllama_math's matmul shape): the splitter's chunk count over the @@ -387,7 +388,7 @@ def perm_wanted(spec, suffix : string) : bool { def time_rows(var stats : array; rounds : int; blk : block<(row : int) : void>) { for (row in range(length(stats))) { stats[row].us |> reserve(rounds) - for (_w in range(3)) { + for (_w in range(SOLO_BURN)) { invoke(blk, row) } } diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index 5b4c8ef433..e2fc7c5894 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -961,11 +961,11 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice if (grouped) { let gq = qp + sliceQ + g * mr * qrow + sbi * qsb * mr let gs = sp + sliceS + g * mr * srow + sbi * dssb * mr - if (fmt == KqFmt.iq3s) { // 26 uniform 4-byte columns over [qs][qh][signs] + if (fmt == KqFmt.iq3s) { // 26 uniform 4-qhbyte columns over [qs][qh][signs] for (m in range64(104l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.iq3xxs) { // 24 uniform 4-byte columns over [qs][aux] + } elif (fmt == KqFmt.iq3xxs) { // 24 uniform 4-qhbyte columns over [qs][aux] for (m in range64(96l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } @@ -991,15 +991,15 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.iq2s) { // 18 uniform 4-byte columns over [idx][signs][qh] + } elif (fmt == KqFmt.iq2s) { // 18 uniform 4-qhbyte columns over [idx][signs][qh] for (m in range64(72l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.iq2xs) { // 16 uniform 4-byte columns over the u16 qs words + } elif (fmt == KqFmt.iq2xs) { // 16 uniform 4-qhbyte columns over the u16 qs words for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.iq2xxs) { // 16 uniform 4-byte columns over the u16 qs words + } elif (fmt == KqFmt.iq2xxs) { // 16 uniform 4-qhbyte columns over the u16 qs words for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } @@ -1016,12 +1016,12 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (p in range64(64l)) { let half = (p % 32l) / 16l let j = (p % 16l) / 4l - var byte = 0u + var qhbyte = 0u for (bb in range64(4l)) { let col = ((p / 32l) * 4l + bb) * 2l + half - byte |= ((uint(gq[128l * mr + (col * mr + r) * 4l + p % 4l]) >> uint(2l * j)) & 3u) << uint(2l * bb) + qhbyte |= ((uint(gq[128l * mr + (col * mr + r) * 4l + p % 4l]) >> uint(2l * j)) & 3u) << uint(2l * bb) } - dq[128l + p] = uint8(byte) + dq[128l + p] = uint8(qhbyte) } } if (fmt == KqFmt.k6 || fmt == KqFmt.k3) { diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index 868c319512..1bd8bf100a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -500,7 +500,7 @@ def public get_matmul_min_chunk_rows_gemv() : int { return g_matmul_min_chunk_ro // GEMV tail: the one-chunk-per-lane equal split leaves the join waiting out the SLOWEST lane's // whole chunk (median lane idles 14-24% of every GEMV stage, per-lane trace 3990X). >1 publishes -// wave-aligned chunks per lane so finished lanes keep serving; default 8; 1 = historical equal split. +// wave-aligned chunks per lane so finished lanes keep serving (only bites past 64*waves*lanes rows); 1 = equal split. var g_gemv_chunks_per_lane = 8 def public set_gemv_chunks_per_lane(v : int) { g_gemv_chunks_per_lane = max(v, 1) } def public get_gemv_chunks_per_lane() : int { return g_gemv_chunks_per_lane } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index df27e5c492..a79877b359 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -1057,6 +1057,7 @@ def private dequant_iq4xs_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, } } +//! w = (d * sc16) * (q - 4) [arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def private dequant_k3_row_grp(kqg : uint8 const?; ksg : uint8 const?; r, mr, nsb : int64; var dst : float?) { unsafe { diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 16d0ac572f..46f5368c03 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -1190,7 +1190,7 @@ def q8q8_tile_s16_gen(var yp : float?; wg : int8 const?; sg : uint16 const?; xqp tune_perm(dot = "vpdpbusd", width = 256, mr = 8, kstep = 2, gkstep = 2, bias = 128, requires = "avxvnni|avx512vnni"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, requires = "avx512vnni,avx512bw"), tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, requires = "avx512vnni,avx512bw"), - tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, pf = 2048, requires = "avx512vnni,avx512bw"), // the crown with a 2 KB software prefetch on the gemv stream (lab row) + tune_perm(dot = "vpdpbusd", width = 512, mr = 16, kstep = 2, gkstep = 2, bias = 128, prefetch = 2048, requires = "avx512vnni,avx512bw"), // the crown with a 2 KB software prefetch on the gemv stream (lab row) tune_perm(dot = "vpdpbssd", width = 256, mr = 8, kstep = 2, requires = "avxvnniint8"), tune_perm(dot = "smmla", mr = 4, kstep = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 4, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 2, nrsplit = 2, requires = "i8mm"), tune_perm(dot = "smmla", mr = 8, kstep = 4, nrsplit = 2, requires = "i8mm"), diff --git a/modules/dasLLAMA/dasllama/dasllama_repack.das b/modules/dasLLAMA/dasllama/dasllama_repack.das index c3fc143ecb..cdaa01fd61 100644 --- a/modules/dasLLAMA/dasllama/dasllama_repack.das +++ b/modules/dasLLAMA/dasllama/dasllama_repack.das @@ -330,12 +330,12 @@ def repack_k6_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { for (blk in range64(8l)) { // qh re-packed per sub-block for (half in range64(2l)) { for (t in range64(4l)) { - var byte = 0u + var qhbyte = 0u for (j in range64(4l)) { let l = (blk / 4l) * 32l + half * 16l + j * 4l + t - byte |= ((uint(tqp[sq + 128l + l]) >> uint(2l * (blk % 4l))) & 3u) << uint(2l * j) + qhbyte |= ((uint(tqp[sq + 128l + l]) >> uint(2l * (blk % 4l))) & 3u) << uint(2l * j) } - kq[dq + 128l * mr + ((blk * 2l + half) * mr + r) * 4l + t] = uint8(byte) + kq[dq + 128l * mr + ((blk * 2l + half) * mr + r) * 4l + t] = uint8(qhbyte) } } } @@ -441,6 +441,7 @@ def repack_iq4xs_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { delete ts } +//! scales interleave as k6's row ([16 x mr int8][mr x f16 d]) [arch(at = "../ARCHITECTURE_CPU_KERNELS.md#kq-subblock-planes")] def repack_k3_grp(var kq : uint8?; var ks : uint8?; n, d, mr : int64) { let nsb = n / 256l From a9312709e11261b0834c80c73459cb56a6650007 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 18:23:07 -0700 Subject: [PATCH 116/123] TDD + woodpecker round: the lattice tables and the GPU gather get silicon-independent gates; the companion own-seat and class-chain scenarios; the GEP scanner; the cache key folds vbmi and amx; a retune always writes the gemv companion; the prefetch pointer drops inbounds; the ladder reference follows NTOK New gates: test_kquant round-trips every grid entry through the vbmi code planes and alphabet and pm1_of against a scalar expansion (pure host data - CI-provable with no VBMI silicon), and proves moe_gpu_gather_stack_kq's own invariant (disk-order and grp planes gather byte-identical device planes, k3 and k6 - the one sub-block reader with no dot oracle). llvm_tune_manifest gains the companion own-entry scenario (write-own -> KV 1 while the main keeps k2); llvm_tune_profiles asserts the box's class heads its ladder; llvm_gep_inbounds.das machine-enforces the GEP rule; test_dispatch_shaping pins the shipped grain default; cpu_supports gains the avx512vbmi implication row. jit_env_salt folds avx512vbmi and amx (tune_cpu_class selects grid decode forms, so both are emission inputs the cache key missed). The seat write is now unconditional - upsert-only merges could keep a stale gemv seat a re-race no longer wants. Emitter pin recomputed (0x6a0cebf74a395071). followup_general 69: a class profile can carry winners a member cannot run (the Cascade Lake case). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0136vDWNJ2GZFzxToEQxWj9i --- .../dasLLAMA/dasllama/dasllama_gemm_gen.das | 8 +- modules/dasLLAMA/dasllama/dasllama_layout.das | 10 +- modules/dasLLAMA/followup_general.md | 11 ++ modules/dasLLAMA/harness/gen_tune_probe.das | 6 +- modules/dasLLAMA/harness/kernel_ladder.sh | 2 +- .../dasLLAMA/tests/test_dispatch_shaping.das | 1 + modules/dasLLAMA/tests/test_kquant.das | 139 ++++++++++++++++++ modules/dasLLVM/daslib/llvm_jit_common.das | 2 +- modules/dasLLVM/daslib/llvm_jit_run.das | 8 +- modules/dasLLVM/tests/llvm_gep_inbounds.das | 49 ++++++ modules/dasLLVM/tests/llvm_tune_manifest.das | 16 ++ .../tests/llvm_tune_manifest_client.das | 8 +- modules/dasLLVM/tests/llvm_tune_profiles.das | 1 + tests/language/cpu_supports.das | 3 + 14 files changed, 249 insertions(+), 15 deletions(-) create mode 100644 modules/dasLLVM/tests/llvm_gep_inbounds.das diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index a3e7d17295..e2640f7f5f 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -452,7 +452,7 @@ def private bias_acc_init(var te : TileEmit; var bi, xlo, xhi : LLVMOpaqueValue? //! llvm.prefetch(read, high locality, data) of the weight plane at byte offset `off` def private prefetch_wg(var te : TileEmit; var off : LLVMOpaqueValue?) { let b = te.builder - var p = LLVMBuildGEP2(b, te.types.t_int8, te.wg, off, "") + var p = LLVMBuildGEP2(b, te.types.t_int8, te.wg, off, "", false) // the look-ahead runs past the plane's end by design - inbounds would be poison var args <- [p, te.types->ConstI32(0ul), te.types->ConstI32(3ul), te.types->ConstI32(1ul)] LLVMBuildCall2(b, te.prefetch_ty, te.prefetch_decl, args, "") } @@ -1503,7 +1503,7 @@ def private grid_rows_path(te : TileEmit) : bool { } //! eight +-1 bytes for sign byte `sb`, lane k negative when bit k is set -def private pm1_of(sb : int) : uint64 { +def public pm1_of(sb : int) : uint64 { // test_kquant round-trips the sign expansion var v = 0ul for (k in range(8)) { v |= uint64(((sb >> k) & 1) != 0 ? 0xFF : 0x01) << uint64(8 * k) @@ -1877,7 +1877,7 @@ def private vbmi_push_iq2(te : TileEmit; var p0, p1 : array; w } //! the grid's symbol alphabet: the halved iq3xxs magnitudes, iq3s's odd magnitudes, the iq2 family's three -def private vbmi_alphabet(kq : int) : array { +def public vbmi_alphabet(kq : int) : array { // test_kquant round-trips the code planes if (kq == 34) return <- [2, 6, 10, 14, 18, 22, 26, 31] if (kq == 33) return <- [1, 3, 5, 7, 9, 11, 13, 15] return <- [8, 25, 43] @@ -1908,7 +1908,7 @@ def private vbmi_tag(kq : int) : string => kq == 25 ? "iq2xxs" : (kq == 24 ? "iq def private vbmi_entries(kq : int) : int => (kq == 25 || kq == 34) ? 256 : (kq == 23 ? 1024 : 512) //! a grid word's low `nbytes` weight bytes as packed symbols (weight t at bits t*bits); a byte outside the alphabet is a broken table -def private vbmi_pack_word(word : uint; alpha : array; bits, nbytes : int) : int { +def public vbmi_pack_word(word : uint; alpha : array; bits, nbytes : int) : int { var code = 0 for (t in range(nbytes)) { let wb = int((word >> uint(8 * t)) & 0xFFu) diff --git a/modules/dasLLAMA/dasllama/dasllama_layout.das b/modules/dasLLAMA/dasllama/dasllama_layout.das index e2fc7c5894..2be0a52f39 100644 --- a/modules/dasLLAMA/dasllama/dasllama_layout.das +++ b/modules/dasLLAMA/dasllama/dasllama_layout.das @@ -961,11 +961,11 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice if (grouped) { let gq = qp + sliceQ + g * mr * qrow + sbi * qsb * mr let gs = sp + sliceS + g * mr * srow + sbi * dssb * mr - if (fmt == KqFmt.iq3s) { // 26 uniform 4-qhbyte columns over [qs][qh][signs] + if (fmt == KqFmt.iq3s) { // 26 uniform 4-byte columns over [qs][qh][signs] for (m in range64(104l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.iq3xxs) { // 24 uniform 4-qhbyte columns over [qs][aux] + } elif (fmt == KqFmt.iq3xxs) { // 24 uniform 4-byte columns over [qs][aux] for (m in range64(96l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } @@ -991,15 +991,15 @@ def moe_gpu_gather_stack_kq(t : Model; fmt : KqFmt; woff : int64; n, rows, slice for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.iq2s) { // 18 uniform 4-qhbyte columns over [idx][signs][qh] + } elif (fmt == KqFmt.iq2s) { // 18 uniform 4-byte columns over [idx][signs][qh] for (m in range64(72l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.iq2xs) { // 16 uniform 4-qhbyte columns over the u16 qs words + } elif (fmt == KqFmt.iq2xs) { // 16 uniform 4-byte columns over the u16 qs words for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } - } elif (fmt == KqFmt.iq2xxs) { // 16 uniform 4-qhbyte columns over the u16 qs words + } elif (fmt == KqFmt.iq2xxs) { // 16 uniform 4-byte columns over the u16 qs words for (m in range64(64l)) { dq[m] = gq[((m / 4l) * mr + r) * 4l + m % 4l] } diff --git a/modules/dasLLAMA/followup_general.md b/modules/dasLLAMA/followup_general.md index 293fb9aed3..37f23090ac 100644 --- a/modules/dasLLAMA/followup_general.md +++ b/modules/dasLLAMA/followup_general.md @@ -867,3 +867,14 @@ at 10 lanes), so the sdot gate is already right. What would actually lift iq2xs on ARM is a cheaper 9-bit index path in the row form (its u16 word costs two column-byte reads per site where iq2xxs reads one), or an ARM analog of the x86 symbol lattice over SMMLA - both open kernel work, not a gate flip. + +69. **A class profile can carry winners a class member cannot run.** `tune_cpu_class()` puts every + AVX512VNNI+BW host in `x86-vnni512`, and that class's shipped profile now carries `grid_vbmi` + winners minted on zen4 (which has VBMI). On a member without VBMI (Cascade Lake, Ice Lake-SP + predecessors), adoption marks the profile complete - nothing races - and every vbmi entry + declines at stamp time to its `fallback=` chain, silently costing those boxes the grid formats' + best seats. Two candidate fixes, unbuilt: split a `x86-vbmi512` class above `x86-vnni512` in + `tune_cpu_class` / `tune_class_chain` (a re-mint renames the profile), or make `profile_try_adopt` + race any family whose adopted winner names a perm `tune_requires_ok` rejects locally (general, + no rename, needs the suffix -> requires mapping surfaced to the adopt layer). The woodpecker + round raised the Cascade Lake case. diff --git a/modules/dasLLAMA/harness/gen_tune_probe.das b/modules/dasLLAMA/harness/gen_tune_probe.das index ee57704d2e..5256eecc72 100644 --- a/modules/dasLLAMA/harness/gen_tune_probe.das +++ b/modules/dasLLAMA/harness/gen_tune_probe.das @@ -2230,8 +2230,10 @@ def tune_mode_run : bool { // nolint:STYLE037,STYLE038 — flat family-by-family // body so the scope still COMPLETES — a missing key re-tunes every start pending[entry] = empty(w) ? "reference" : w print("{entry} winner: {empty(w) ? "reference (gate-failure fallback)" : w}\n") - if (!empty(gw)) { // the gemv's own seat (same layout as the tile's); absent = the companion follows the tile - pending[kq_gemv_entry(fmt)] = gw + // ALWAYS written: tune_sidecar_merge is upsert-only, so a seat the re-race no longer wants + // must be overwritten with the follow-the-tile spelling, not left standing + pending[kq_gemv_entry(fmt)] = empty(gw) ? (empty(w) ? "reference" : w) : gw + if (!empty(gw)) { print("{kq_gemv_entry(fmt)} winner: {gw} (the tile seat {w} keeps the tile)\n") } g_rows[entry] = JV(krows) diff --git a/modules/dasLLAMA/harness/kernel_ladder.sh b/modules/dasLLAMA/harness/kernel_ladder.sh index 47f135ed3b..92893d80b6 100644 --- a/modules/dasLLAMA/harness/kernel_ladder.sh +++ b/modules/dasLLAMA/harness/kernel_ladder.sh @@ -75,7 +75,7 @@ fi for f in $(cut -f1 "$WORK/ours.tsv" | sort -u); do t=$(ggml_type "$f") [ -n "$t" ] || { echo "kernel_ladder: no ggml type for '$f'" >&2; continue; } - GGML_BENCH_THREADS=$THREADS "$TBO" perf -b CPU -o MUL_MAT -p "type_a=$t,type_b=f32,m=$ROWS,n=(1|512)," 2>/dev/null \ + GGML_BENCH_THREADS=$THREADS "$TBO" perf -b CPU -o MUL_MAT -p "type_a=$t,type_b=f32,m=$ROWS,n=(1|$NTOK)," 2>/dev/null \ | sed 's/\x1b\[[0-9;]*m//g' | awk -v f="$f" ' /MUL_MAT\(/ { n = $0; sub(/.*,n=/, "", n); sub(/,.*/, "", n) diff --git a/modules/dasLLAMA/tests/test_dispatch_shaping.das b/modules/dasLLAMA/tests/test_dispatch_shaping.das index 8f1f14af12..2aa895044b 100644 --- a/modules/dasLLAMA/tests/test_dispatch_shaping.das +++ b/modules/dasLLAMA/tests/test_dispatch_shaping.das @@ -19,6 +19,7 @@ def private pin_shaping_baseline() { set_batch_lane_cap(0) set_matmul_min_chunk_rows(16) set_matmul_min_chunk_rows_gemv(64) + t |> equal(get_gemv_chunks_per_lane(), 8, "the shipped default - retune the ladder tables if this moves") set_gemv_chunks_per_lane(1) set_gemv_wave_fill(true) set_batch_grid_2d(0) diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index c1bf709a7a..5e01168dcb 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -6,6 +6,8 @@ options _function_length = 300 // and each gate carries its whole fixture options _dasllama_internal = true require dastest/testing_boost public +require dasllama/dasllama_gemm_gen // vbmi_pack_word / vbmi_alphabet / pm1_of - the lattice's pure table builders +require dasllama/dasllama_layout // moe_gpu_gather_stack_kq - the device-plane gather oracle require _model_tier require dasllama/dasllama_gguf require dasllama/dasllama_common // dequant_kq_plane_sb - the carrier-form dispatcher @@ -2564,3 +2566,140 @@ def test_kq_plane_sb_metal_blob_form(t : T?) { } } } + +// The VBMI lattice's tables are pure host-side data - prove them on any box, no VBMI needed: +// every grid entry's packed code decodes back to the grid's own bytes through the alphabet, +// and the ksigns +-1 expansion matches a scalar re-derivation. +def private vbmi_plane_gate(t : T?; kq : int) { + var alpha <- vbmi_alphabet(kq) + let iq3 = kq == 33 || kq == 34 + let bits = iq3 ? 3 : 2 + let nbytes = iq3 ? 2 : 4 + let mask = iq3 ? 7 : 3 + var words : array + if (kq == 34) { + words <- [for (w in iq3xxs_grid()); w] + } elif (kq == 33) { + words <- [for (w in iq3s_grid()); w] + } elif (kq == 25) { + words <- [for (w in iq2xxs_grid2()); w] + } elif (kq == 24) { + words <- [for (w in iq2xs_grid2()); w] + } else { + words <- [for (w in iq2s_grid2()); w] + } + var bad = 0 + for (word in words) { + for (half in range(iq3 ? 2 : 1)) { // iq3: one word = two plane bytes (low/high pair); iq2: one word = one plane byte + let w = half == 0 ? word : (word >> 16u) + let code = vbmi_pack_word(w, alpha, bits, nbytes) + for (tb in range(nbytes)) { + let sym = (code >> (bits * tb)) & mask + let back = alpha[sym] + let orig = int((w >> uint(8 * tb)) & 0xFFu) + if (back != orig) { + bad++ + } + } + } + } + t |> equal(bad, 0, "kq {kq}: every grid byte round-trips through the code plane and alphabet") +} + +[test] +def test_vbmi_lattice_tables(t : T?) { + for (kq in [25, 24, 23, 34, 33]) { + t |> run("vbmi code planes round-trip the kq {kq} grid") @(t : T?) { + vbmi_plane_gate(t, kq) + } + } + t |> run("pm1_of matches the scalar +-1 expansion of every ksigns byte") @(tt : T?) { + var bad = 0 + for (sb in range(256)) { + let packed = pm1_of(sb) + for (k in range(8)) { + let lane = int8(int(packed >> uint64(8 * k)) & 0xFF) + let want = (sb & (1 << k)) != 0 ? int8(-1) : int8(1) + if (lane != want) { + bad++ + } + } + } + tt |> equal(bad, 0) + } +} + +// The GPU device-plane gather's own invariant ("device bytes are identical either way"): gathering +// from disk-order planes and from the grp-repacked planes must produce byte-identical wq/ws. +// This is the only k3/k6 sub-block-plane reader with no dot-product oracle - the byte equality IS its gate. +def private kq_gpu_gather_gate(t : T?; fmt : int) { + let n = 512l + let d = 32l + let nsb = n / 256l + let mr = kq_layout_of(fmt) + let qsb = kq_qsb(fmt) + let ssb = kq_ssb(fmt) + let blk1 <- fmt == 3 ? build_q3k_block() : build_q6k_block() + var kq : array + var ks : array + kq |> resize(d * nsb * qsb) + ks |> resize(d * nsb * ssb) + for (r in range64(d)) { + for (sbi in range64(nsb)) { + if (fmt == 3) { + transcode_q3k_superblock(blk1, 0l, kq, (r * nsb + sbi) * qsb, ks, (r * nsb + sbi) * ssb) + } else { + transcode_q6k_superblock(blk1, 0l, kq, (r * nsb + sbi) * qsb, ks, (r * nsb + sbi) * ssb) + } + } + var st = 0xC0FFEEu + uint(r) * 0x9E3779Bu + for (i in range64(nsb * qsb)) { + st = st * 1664525u + 1013904223u + kq[r * nsb * qsb + i] = uint8(uint(kq[r * nsb * qsb + i]) ^ (st >> 16u)) + } + } + var m = Model() + if (fmt == 3) { + m.k3q := kq + m.k3s := ks + } else { + m.k6q := kq + m.k6s := ks + } + var wqDisk : array + var wsDisk : array + moe_gpu_gather_stack_kq(m, fmt == 3 ? KqFmt.k3 : KqFmt.k6, 0l, n, d, d, false, mr, wqDisk, wsDisk) + unsafe { + if (fmt == 3) { + repack_k3_grp(addr(m.k3q[0]), addr(m.k3s[0]), n, d, mr) + } else { + repack_k6_grp(addr(m.k6q[0]), addr(m.k6s[0]), n, d, mr) + } + } + var wqGrp : array + var wsGrp : array + moe_gpu_gather_stack_kq(m, fmt == 3 ? KqFmt.k3 : KqFmt.k6, 0l, n, d, d, true, mr, wqGrp, wsGrp) + var qdiff = 0 + for (a, b in wqDisk, wqGrp) { + if (a != b) { + qdiff++ + } + } + var sdiff = 0 + for (a, b in wsDisk, wsGrp) { + if (a != b) { + sdiff++ + } + } + t |> equal(qdiff, 0, "kq {fmt}: quant device bytes identical from disk-order and grp planes") + t |> equal(sdiff, 0, "kq {fmt}: scale device bytes identical from disk-order and grp planes") +} + +[test] +def test_kq_gpu_gather_grp(t : T?) { + for (fmt in [3, 6]) { + t |> run("moe_gpu_gather_stack_kq kq {fmt}: disk-order and grp planes gather identically") @(t : T?) { + kq_gpu_gather_gate(t, fmt) + } + } +} diff --git a/modules/dasLLVM/daslib/llvm_jit_common.das b/modules/dasLLVM/daslib/llvm_jit_common.das index b84644aaf3..908f849619 100644 --- a/modules/dasLLVM/daslib/llvm_jit_common.das +++ b/modules/dasLLVM/daslib/llvm_jit_common.das @@ -430,7 +430,7 @@ var public g_target_x64_f16c = false var public g_target_x64_vnni256 = false var public g_target_x64_avx512bw = false var public g_target_x64_avx512vnni = false -var public g_target_x64_avx512vbmi = false //! zmm byte permutes across two registers (VPERMI2B) + VPMULTISHIFTQB: the grid formats' symbol lattice +var public g_target_x64_avx512vbmi = false //! zmm byte permutes across two registers (VPERMI2B) + VPMULTISHIFTQB // AVX-VNNI-INT8 (VEX vpdpbssd — native s8×s8, no sign trick). No shipped silicon in the fleet // yet; lights up via cpuid on future boxes and via DAS_JIT_X64_FORCE_FEATURES for emission-only // verification today. diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index a0e945b367..69cb9c1274 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -40,7 +40,7 @@ let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x68ul // the grid formats' gemv appli // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) -let LLVM_JIT_EMITTER_HASH : uint64 = 0x26d3a061cfe6fc27ul +let LLVM_JIT_EMITTER_HASH : uint64 = 0x6a0cebf74a395071ul let JIT_FNV_PRIME : uint64 = 1099511628211ul @@ -230,6 +230,12 @@ def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_ if (cpu_supports("avx512vnni")) { isa |= 16ul } + if (cpu_supports("avx512vbmi")) { // the grid formats' symbol lattice emits on it (grid = "vbmi") + h = (h ^ 0x1Bul) * JIT_FNV_PRIME + } + if (cpu_supports("amx-tile") && cpu_supports("amx-int8")) { // tune_cpu_class picks x86-amx, which selects grid decode forms + h = (h ^ 0x1Cul) * JIT_FNV_PRIME + } if (cpu_supports("avxvnniint8")) { isa |= 32ul } diff --git a/modules/dasLLVM/tests/llvm_gep_inbounds.das b/modules/dasLLVM/tests/llvm_gep_inbounds.das new file mode 100644 index 0000000000..8ec4148567 --- /dev/null +++ b/modules/dasLLVM/tests/llvm_gep_inbounds.das @@ -0,0 +1,49 @@ +options gen2 + +require dastest/testing_boost public +require daslib/fio +require strings +require daslib/strings_boost +require daslib/module_path + +// Machine-enforces REVIEW.md's GEP rule: no dasLLVM source outside bindings/ may call +// LLVMSetIsInBounds - a constant-folded GEP is a ConstantExpr, and the setter's cast writes +// the flag through the wrong type into it (ARCHITECTURE.md#gep-constant-fold). Build the GEP +// in-bounds with LLVMBuildInBoundsGEP2 instead. Weakening this test is a defect. + +def private scan_dir(dir : string; var offenders : array) { + fio::dir(dir) $(name : string) { + if (name == "." || name == "..") { + return + } + let path = "{dir}/{name}" + let st = stat(path) + if (st.is_dir) { + if (name != "bindings") { + scan_dir(path, offenders) + } + return + } + if (!(name |> ends_with(".das"))) { + return + } + let text = fread(path) + var ln = 1 + for (line in split(text, "\n")) { + if (line |> find("LLVMSetIsInBounds") >= 0) { + offenders |> push("{path}:{ln}") + } + ln++ + } + } +} + +[test] +def test_no_llvm_set_is_in_bounds(t : T?) { + var offenders : array + scan_dir("{get_this_module_dir()}/..", offenders) + for (o in offenders) { + t |> failure("LLVMSetIsInBounds called at {o} - build the GEP in-bounds with LLVMBuildInBoundsGEP2 (REVIEW.md; ARCHITECTURE.md#gep-constant-fold)") + } + t |> success(empty(offenders), "no LLVMSetIsInBounds call sites outside bindings/") +} diff --git a/modules/dasLLVM/tests/llvm_tune_manifest.das b/modules/dasLLVM/tests/llvm_tune_manifest.das index 53f314459b..eb743dfcd7 100644 --- a/modules/dasLLVM/tests/llvm_tune_manifest.das +++ b/modules/dasLLVM/tests/llvm_tune_manifest.das @@ -95,6 +95,22 @@ def test_llvm_tune_manifest_roundtrip(t : T?) { // nolint:STYLE038 - flat sequ t |> success(other6) // no sidecar entry -> the fallback stamp (k1) stands t |> success(ref5) // explicit "reference" entry -> the original body despite fallback + // the companion's OWN sidecar entry (the gemv-seat rail): man_kv stamps from it while man_add keeps k2 + var olines : array + let rco = spawn_client("\"{bin}\" -jit \"{clientPath}\" -- write-own", olines) + t |> equal(rco, 0) + var ownLines : array + let rcor = spawn_client("\"{bin}\" -jit \"{clientPath}\"", ownLines) + t |> equal(rcor, 0) + var ownResult7 = false + var ownKv1 = false + for (ln in ownLines) { + ownResult7 ||= ln == "RESULT 7" + ownKv1 ||= ln == "KV 1" + } + t |> success(ownResult7, "the main entry still stamps k2") + t |> success(ownKv1, "the companion stamps from its own entry") + // STALE sidecar (older than the daslang binary) reads as absent: back to fallback tier let backRes = set_mtime_result(sidecarPath, mktime(2000, 1, 1, 0, 0, 0)) t |> success(!(backRes is error), "could not back-date the sidecar") diff --git a/modules/dasLLVM/tests/llvm_tune_manifest_client.das b/modules/dasLLVM/tests/llvm_tune_manifest_client.das index c27d7e70ba..750320fab5 100644 --- a/modules/dasLLVM/tests/llvm_tune_manifest_client.das +++ b/modules/dasLLVM/tests/llvm_tune_manifest_client.das @@ -40,10 +40,16 @@ def main { var n = 2 // nolint:LINT003 — var defeats const-fold so the calls survive to codegen let args <- get_command_line_arguments() // locked view — do not delete var doWrite = false + var doWriteOwn = false for (a in args) { doWrite ||= a == "write" + doWriteOwn ||= a == "write-own" } - if (doWrite) { + if (doWriteOwn) { + // the companion's OWN entry (the gemv-seat shape): man_kv stamps from it, man_add keeps its k2 + let ok = tune_manifest_set("man_kv", "k1") + print("WRITE {ok ? "ok" : "FAILED"} -> {tune_manifest_path()}\n") + } elif (doWrite) { let ok = tune_manifest_set("man_add", "k2") && tune_manifest_set("man_ref", "reference") print("WRITE {ok ? "ok" : "FAILED"} -> {tune_manifest_path()}\n") } else { diff --git a/modules/dasLLVM/tests/llvm_tune_profiles.das b/modules/dasLLVM/tests/llvm_tune_profiles.das index c528c2d0f2..bf80fd6253 100644 --- a/modules/dasLLVM/tests/llvm_tune_profiles.das +++ b/modules/dasLLVM/tests/llvm_tune_profiles.das @@ -137,6 +137,7 @@ def test_llvm_tune_profiles(t : T?) { // nolint:STYLE038 - one flat spawn-and-as // 6. the class chain: only a LOWER class's profile on disk, and the box adopts it (the // residue race then covers whatever its own class unlocks) var chain <- tune_class_chain() + t |> equal(chain[0], tune_cpu_class(), "the box's own class heads its ladder - a class outside the chain could never adopt its own profile") if (length(chain) > 1) { let lower = chain[length(chain) - 1] remove(sidecar) diff --git a/tests/language/cpu_supports.das b/tests/language/cpu_supports.das index 1f406e783b..6f15ac6388 100644 --- a/tests/language/cpu_supports.das +++ b/tests/language/cpu_supports.das @@ -25,6 +25,9 @@ def test_cpu_supports(t : T?) { if (cpu_supports("avx512vnni")) { t |> success(cpu_supports("avx512f"), "avx512vnni implies avx512f") } + if (cpu_supports("avx512vbmi")) { + t |> success(cpu_supports("avx512f"), "avx512vbmi implies avx512f") + } if (cpu_supports("avx512bw")) { t |> success(cpu_supports("avx512f"), "avx512bw implies avx512f") } From a3cfd7dc650d84732aa54832661b5551c8688f05 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 18:41:15 -0700 Subject: [PATCH 117/123] test fixes from the first run: the GEP scanner self-excludes and gets its module_path require; the grain-default pin moves into its own [test] --- modules/dasLLAMA/tests/test_dispatch_shaping.das | 6 +++++- modules/dasLLVM/tests/llvm_gep_inbounds.das | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/dasLLAMA/tests/test_dispatch_shaping.das b/modules/dasLLAMA/tests/test_dispatch_shaping.das index 2aa895044b..536f105a4e 100644 --- a/modules/dasLLAMA/tests/test_dispatch_shaping.das +++ b/modules/dasLLAMA/tests/test_dispatch_shaping.das @@ -19,12 +19,16 @@ def private pin_shaping_baseline() { set_batch_lane_cap(0) set_matmul_min_chunk_rows(16) set_matmul_min_chunk_rows_gemv(64) - t |> equal(get_gemv_chunks_per_lane(), 8, "the shipped default - retune the ladder tables if this moves") set_gemv_chunks_per_lane(1) set_gemv_wave_fill(true) set_batch_grid_2d(0) } +[test] +def test_gemv_chunks_default(t : T?) { + t |> equal(get_gemv_chunks_per_lane(), 8, "the shipped default - retune the ladder tables if this moves") +} + [test] def test_lanes_for_work(t : T?) { with_job_que() { diff --git a/modules/dasLLVM/tests/llvm_gep_inbounds.das b/modules/dasLLVM/tests/llvm_gep_inbounds.das index 8ec4148567..0131f29b12 100644 --- a/modules/dasLLVM/tests/llvm_gep_inbounds.das +++ b/modules/dasLLVM/tests/llvm_gep_inbounds.das @@ -24,7 +24,7 @@ def private scan_dir(dir : string; var offenders : array) { } return } - if (!(name |> ends_with(".das"))) { + if (!(name |> ends_with(".das")) || name == "llvm_gep_inbounds.das") { // the scanner names its own quarry return } let text = fread(path) From 73721d04e1d5692c9d1582b1d6949f935a80504a Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 20:07:53 -0700 Subject: [PATCH 118/123] popen_argv respells argv[0] natively on Windows: CreateProcess rejected a relative forward-slash exe, which is exactly what the board rig spawned The rig's parity pregate "flake" was a spawn that never happened: with a NULL application name, CreateProcess refuses "modules/.../dasllama-bench.exe" while "./x.exe", "../x.exe", absolute and backslash spellings all resolve. Every caller of popen_argv, spawn_argv and popen_argv_pipe now gets argv[0] respelled before the command line is built; the caller-side backslash workaround leaves the fio doc and the language skill; tests/fio/popen_argv.das spawns the offending shape; run_and_stream logs a failed spawn instead of pointing at output that never came. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014sNMfrttRvpmQypECWoV3x --- daslib/fio.das | 4 ++-- .../dasLLAMA/performance/profile_common.das | 1 + skills/daslang/references/files-and-paths.md | 4 ++-- src/builtin/module_builtin_fio.cpp | 10 ++++++++- tests/README.md | 2 +- tests/fio/popen_argv.das | 22 +++++++++++++++++++ 6 files changed, 37 insertions(+), 6 deletions(-) diff --git a/daslib/fio.das b/daslib/fio.das index 65d1a06be1..db1d3311e2 100644 --- a/daslib/fio.das +++ b/daslib/fio.das @@ -692,8 +692,8 @@ def rmdir_rec_result(path : string) : fs_result_bool { def run_and_capture(args : array; var output : string&; timeout_sec : float = 0.0) : int { //! Run an external command and capture its stdout+stderr (merged into one pipe by the underlying ``popen_argv``). Returns the process exit code; - //! -1 means the spawn itself failed. No shell is involved, but on Windows the program path (``args[0]``) must use backslashes — - //! CreateProcess does not resolve ``bin/daslang``-style forward-slash relative paths (probe-verified); ``replace(exe, "/", "\\")`` first. + //! -1 means the spawn itself failed. No shell is involved; the program path (``args[0]``) may use either slash style on every + //! platform (``popen_argv`` respells it natively before the Windows CreateProcess call, which rejects ``bin/daslang``-style forward-slash relative paths). var captured : string let exit_code = unsafe(popen_argv(args, timeout_sec, $(f) { if (f != null) { diff --git a/modules/dasLLAMA/performance/profile_common.das b/modules/dasLLAMA/performance/profile_common.das index 94e6dc56ef..fe6e52e8d4 100644 --- a/modules/dasLLAMA/performance/profile_common.das +++ b/modules/dasLLAMA/performance/profile_common.das @@ -315,6 +315,7 @@ def run_and_stream(args : array; var tail_out : string&; keep_tail : int var acc = "" let rc = unsafe(popen_argv(args, 0.0, $(f) { if (f == null) { + to_log(LOG_ERROR, "run_and_stream: could not spawn {args[0]} (no child, no output)\n") return } while (true) { diff --git a/skills/daslang/references/files-and-paths.md b/skills/daslang/references/files-and-paths.md index 40f66c9983..27592527eb 100644 --- a/skills/daslang/references/files-and-paths.md +++ b/skills/daslang/references/files-and-paths.md @@ -63,8 +63,8 @@ fopen(path, "rb") $(f) { concept-assert raw POD. The array forms **panic above 2 GiB**; `long_fread` / `long_fwrite` return `int64` and have no cap. - `run_and_capture(args, var output, timeout_sec = 0.0) : int` runs a child with no shell, capturing - merged stdout+stderr. **Windows trap: `args[0]` must use backslashes** - the OS will not resolve a - forward-slash relative path - so `replace(exe, "/", "\\")` first. + merged stdout+stderr. `args[0]` may use either slash style on every host - the spawn respells it + natively for Windows, whose CreateProcess rejects a forward-slash relative path. ## Mutating operations and their three error forms diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 15997b8eb1..4b5dc782ce 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -1173,11 +1173,19 @@ namespace das { return out; } + static string winNativeExePath ( const char * exe ) { + string s = exe ? exe : ""; + for ( auto & ch : s ) { + if ( ch == '/' ) ch = '\\'; + } + return s; + } + static string winBuildCommandLine ( char ** argv, uint64_t argc ) { string s; for ( uint64_t i = 0; i < argc; ++i ) { if ( i ) s += ' '; - s += winArgvEscape(argv[i]); + s += i ? winArgvEscape(argv[i]) : winArgvEscape(winNativeExePath(argv[i]).c_str()); } return s; } diff --git a/tests/README.md b/tests/README.md index b794fd6f06..25a1c97599 100644 --- a/tests/README.md +++ b/tests/README.md @@ -384,7 +384,7 @@ Every `.das` file in this directory tree is listed below, grouped by subdirector | fio_file.das | File I/O - fopen, fread, fwrite with fuzzing | | | fio_utils.das | File utilities - fexist, rmdir, rmdir_rec, fread/fwrite by path, get_das_version | | | glob_test.das | Pathname glob - `match_glob` (literal, `*`, `**`, `?`, `[a-z]`, `[!abc]`, edge cases), `glob`, `glob_filtered` walk, `is_glob_pattern` | | -| popen_argv.das | `popen_argv` - basic invocation, non-zero exit on unknown flag, exit code capture | | +| popen_argv.das | `popen_argv` - basic invocation, non-zero exit on unknown flag, exit code capture, a forward-slash relative argv[0] | | ## fixed_array/ diff --git a/tests/fio/popen_argv.das b/tests/fio/popen_argv.das index ec3c9d0023..2a6bab7595 100644 --- a/tests/fio/popen_argv.das +++ b/tests/fio/popen_argv.das @@ -112,6 +112,28 @@ def test_popen_argv_arg_with_embedded_quote(t : T?) { } } +[test] +def test_popen_argv_relative_forward_slash_exe(t : T?) { + t |> run("a relative argv[0] spelled with forward slashes spawns") @(t : T?) { + // "Release/daslang.exe" from bin/ - no leading ./, forward slash: the one spelling + // Windows CreateProcess refuses unless the spawn respells it natively + let exe = get_full_file_name(das_exe()) + let parent = dir_name(exe) + let rel = "{base_name(parent)}/{base_name(exe)}" + let saved = getcwd() + if (!chdir(dir_name(parent))) { + t |> failure("could not chdir to {dir_name(parent)}") + return + } + var output : string + let rc = run_argv([rel, "--help"], output) + t |> success(chdir(saved), "could not restore the working directory {saved}") + t |> equal(rc, 0, "spawning {rel} from {dir_name(parent)} failed") + t |> success(find(output, "daslang version") >= 0, + "expected 'daslang version' in output: {output}") + } +} + [test] def test_popen_argv_child_stdin_isolated(t : T?) { t |> run("child cannot read parent's stdin") @(t : T?) { From eafdd0faff513a71ce8a8eff28dbc85131270e5e Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 20:07:54 -0700 Subject: [PATCH 119/123] dasLLAMA: the zen2 board row for gemma-4-12B Q4_K_M re-minted on the fixed rig - pp 1.18x, tg 1.04x of the clean reference, the ratio the 08-04 row held The row rides the adopted x86-avx2 class profile (snapshot zen2.tune.3fe069fc370e); both engines read about 5% under the 08-04 sweep on this box and the ratio held. The plan ledger carries the spawn defect and the row. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014sNMfrttRvpmQypECWoV3x --- .../dasLLAMA/performance/records/zen2.json | 51 +- .../records/zen2.tune.3fe069fc370e.json | 1496 +++++++++++++++++ plans/kernel_parity_pass.md | 16 + 3 files changed, 1539 insertions(+), 24 deletions(-) create mode 100644 modules/dasLLAMA/performance/records/zen2.tune.3fe069fc370e.json diff --git a/modules/dasLLAMA/performance/records/zen2.json b/modules/dasLLAMA/performance/records/zen2.json index 2aa8b17916..0e759ec63f 100644 --- a/modules/dasLLAMA/performance/records/zen2.json +++ b/modules/dasLLAMA/performance/records/zen2.json @@ -462,37 +462,40 @@ "flavor":"tuned", "box":"zen2", "threads":16, - "date":"2026-08-04", - "cmd":"modules/dasLLAMA/performance/_rig/dasllama-bench/dasllama-bench.exe -- -m D:\\work\\llama.cpp\\models\\gemma-4-12B-it-Q4_K_M.gguf -p 512 -n 128 -r 5 -t 16 -o json --json-path modules/dasLLAMA/performance/records/_cell_zen2.json", + "date":"2026-09-01", + "cmd":"modules\\dasLLAMA\\performance\\_rig\\dasllama-bench\\dasllama-bench.exe -- -m D:/Work/llama.cpp/models/gemma-4-12B-it-Q4_K_M.gguf -p 512 -n 128 -r 5 -t 16 -o json --json-path modules/dasLLAMA/performance/records/_cell_zen2.json", "hardware":{ "cpu":"AMD Ryzen Threadripper 3990X 64-Core Processor", "arch":"AMD64", - "os":"Microsoft Windows [Version 10.0.26200.8875]", + "os":"Microsoft Windows [Version 10.0.26200.9168]", "total_cores":128, "ram_gb":256, "gpu":"NVIDIA GeForce RTX 5060 Ti", - "ram_config":"8×32 GB @ 2666 MT/s", + "ram_config":"8x32 GB @ 2666 MT/s", "power_plan":"High performance", "smt":"on", - "remote_desktop":"off" + "remote_desktop":"parsec" }, "tests":{ "pp512":{ - "tok_s":70.207003056096113, - "stddev":0.25786811113357544 + "tok_s":66.41365246733001, + "stddev":1.7098919153213501 }, "tg128":{ - "tok_s":7.301398548790532, - "stddev":0.009024646133184433 + "tok_s":6.7354839007033362, + "stddev":0.045002911239862442 } }, "source":"official", - "sha":"37a2c1662", - "version":"0.6.4", - "tune":"k4q8_tile_gen=dot_maddubs_width256_mr8 (manifest); k5q8_tile_gen=dot_maddubs_width256_mr8 (manifest); k6q8_tile_gen=dot_maddubs_width256_mr8 (manifest); q40q8_tile_gen=dot_maddubs_width256_mr8 (manifest); q8q8_tile_gen=dot_maddubs_width256_mr8_kstep2 (manifest); q51q8_gemv_gen=mr8 (manifest)", - "tune_sha":"fe4d9873989abfef6cdca7eba5a000845d781cf3c4bb7e7673db5d83335e4ece", + "sha":"6f7da32ed", + "version":"unknown", + "dasllama_version":15, + "tune":"k4q8_tile_gen=dot_maddubs_width256_mr8 (manifest); k5q8_tile_gen=dot_maddubs_width256_mr8 (manifest); k6q8_tile_gen=dot_maddubs_width256_mr8 (manifest); q40q8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq4xsq8_tile_gen=dot_maddubs_width256_mr8 (manifest); k3q8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq3sq8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq3xxsq8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq4nlq8_tile_gen=dot_maddubs_width256_mr8 (manifest); k2q8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq2sq8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq2xsq8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq2xxsq8_tile_gen=dot_maddubs_width256_mr8 (manifest); q8q8_tile_gen=dot_maddubs_width256_mr8_kstep2 (manifest); q51q8_tile_gen=dot_maddubs_width256_mr8 (manifest)", + "tune_sha":"3fe069fc370ef67773722b024397886cce90777ee6859a22df23099b9c9683fb", + "noise":"ok", + "parity":"ok", "exec_fmt":"weights k4/k6/q8 native; q8 activations; f32 scales; planar x64-gen (repacked)", - "env":"DASLLAMA_BOX=zen2 DASLLAMA_MODELS_DIR=D:\\work\\llama.cpp\\models\\", + "env":"DASLLAMA_MODELS_DIR=D:/Work/llama.cpp/models/", "files":[ { "role":"weights", @@ -508,32 +511,32 @@ "flavor":"clean-cpu", "box":"zen2", "threads":16, - "date":"2026-08-04", - "cmd":"D:\\Work\\llama.cpp-ref-ebd048f\\build-clean-cpu\\bin\\Release\\llama-bench.exe -m D:\\work\\llama.cpp\\models\\gemma-4-12B-it-Q4_K_M.gguf -ngl 0 -t 16 -p 512 -n 128 -r 5 -o json", + "date":"2026-09-01", + "cmd":"D:/Work/llama.cpp/build-clean-cpu/bin/Release/llama-bench.exe -m D:/Work/llama.cpp/models/gemma-4-12B-it-Q4_K_M.gguf -ngl 0 -t 16 -p 512 -n 128 -r 5 -o json", "hardware":{ "cpu":"AMD Ryzen Threadripper 3990X 64-Core Processor", "arch":"AMD64", - "os":"Microsoft Windows [Version 10.0.26200.8875]", + "os":"Microsoft Windows [Version 10.0.26200.9168]", "total_cores":128, "ram_gb":256, "gpu":"NVIDIA GeForce RTX 5060 Ti", - "ram_config":"8×32 GB @ 2666 MT/s", + "ram_config":"8x32 GB @ 2666 MT/s", "power_plan":"High performance", "smt":"on", - "remote_desktop":"off" + "remote_desktop":"parsec" }, "tests":{ "pp512":{ - "tok_s":59.069308999999997, - "stddev":0.26938099999999998 + "tok_s":56.404944999999998, + "stddev":0.141788 }, "tg128":{ - "tok_s":6.7950109999999997, - "stddev":0.016497000000000001 + "tok_s":6.4491550000000002, + "stddev":0.035576999999999998 } }, "source":"official", - "sha":"ebd048fc5", + "sha":"6c84c7d5d", "exec_fmt":"native gguf formats; cpu runtime layout repack", "files":[ { diff --git a/modules/dasLLAMA/performance/records/zen2.tune.3fe069fc370e.json b/modules/dasLLAMA/performance/records/zen2.tune.3fe069fc370e.json new file mode 100644 index 0000000000..0ec1afa677 --- /dev/null +++ b/modules/dasLLAMA/performance/records/zen2.tune.3fe069fc370e.json @@ -0,0 +1,1496 @@ +{ + "kernels" : { + "iq4xsq8_gemv_gen" : "dot_maddubs_width256_mr8", + "q40q8_gemv_gen" : "dot_maddubs_width256_mr8", + "add_inplace" : "vec8_u2", + "iq2xsq8_gemv_gen" : "dot_maddubs_width256_mr8", + "cvt_f32_to_f16" : "vec8_u2", + "iq2sq8_gemv_gen" : "dot_maddubs_width256_mr8", + "rope_scaled_neox_tab" : "plain", + "q51q8_tile_gen" : "dot_maddubs_width256_mr8", + "softmax" : "vec8_u2", + "mul_inplace" : "vec8_u2", + "quantize_q8_0_bs_into_ptr" : "vec16_u2", + "dot_q8kv" : "vec4_u4", + "iq2xsq8_tile_gen" : "dot_maddubs_width256_mr8", + "dot_q8q8" : "vec16", + "iq4xsq8_tile_gen" : "dot_maddubs_width256_mr8", + "iq4nlq8_tile_gen" : "dot_maddubs_width256_mr8", + "quantize_q8kv_row" : "plain", + "axpy_f16" : "vec8_u2", + "q40q8_tile_gen" : "dot_maddubs_width256_mr8", + "axpy_tq4kv" : "vec8_u2", + "cvt_tq4kv_to_f32" : "vec8_u2", + "axpy" : "vec8_u2", + "dot_mx4q8" : "u2", + "dot_q8q8kv" : "vec16", + "softmax_sink" : "vec8_u2", + "iq2xxsq8_tile_gen" : "dot_maddubs_width256_mr8", + "dot_q8q8_laneq4x4" : "", + "dot_bf16" : "vec8_u2", + "iq2sq8_tile_gen" : "dot_maddubs_width256_mr8", + "add_scale_inplace" : "vec8_u2", + "cvt_q8kv_to_f32" : "vec8_u2", + "axpy_q8kv" : "vec8_u2", + "dot_q8q8_f16s" : "vec16", + "iq2xxsq8_gemv_gen" : "dot_maddubs_width256_mr8", + "q8q8_tile_gen" : "dot_maddubs_width256_mr8_kstep2", + "quantize_q8_0_into_ptr" : "plain", + "k2q8_tile_gen" : "dot_maddubs_width256_mr8", + "gemm_f32_uk_4x16" : "plain", + "dot_q51e" : "vec16", + "k3q8_tile_gen" : "dot_maddubs_width256_mr8", + "k4q8_tile_gen" : "dot_maddubs_width256_mr8", + "k5q8_tile_gen" : "dot_maddubs_width256_mr8", + "k6q8_tile_gen" : "dot_maddubs_width256_mr8", + "k4q8_gemv_gen" : "dot_maddubs_width256_mr8", + "dot_f16" : "vec8_u2", + "k5q8_gemv_gen" : "dot_maddubs_width256_mr8", + "k6q8_gemv_gen" : "dot_maddubs_width256_mr8", + "cvt_f16_to_f32" : "vec8_u2", + "k3q8_gemv_gen" : "dot_maddubs_width256_mr8", + "iq4nlq8_gemv_gen" : "dot_maddubs_width256_mr8", + "k2q8_gemv_gen" : "dot_maddubs_width256_mr8", + "dot" : "vec8_u2", + "iq3xxsq8_tile_gen" : "dot_maddubs_width256_mr8", + "iq3xxsq8_gemv_gen" : "dot_maddubs_width256_mr8", + "iq3sq8_tile_gen" : "dot_maddubs_width256_mr8", + "dot_q8tq4kv" : "vec16", + "iq3sq8_gemv_gen" : "dot_maddubs_width256_mr8", + "scale_inplace" : "vec8_u2", + "dot_q4" : "vec8_u2", + "copy_floats" : "vec8_u2", + "quantize_tq4kv_row" : "plain", + "rmsnorm" : "vec16_u2" + }, + "provenance" : { + "validation" : "ok", + "noise_probes" : "start cv 0.29%; mid1 cv 0.19%; mid2 cv 0.09%; end cv 0.28%", + "platform" : "windows", + "noise_floor_cv_pct" : "0.29", + "features" : "avx2;f16c;fma;sse4.2", + "box" : "windows|x86_64||10.0.26200.9168|AMD64 Family 23 Model 49 Stepping 0, AuthenticAMD", + "engine_sha" : "6f7da32ed", + "written" : "2026-09-02T01:42:17.414Z", + "validation_demoted" : "0", + "mode" : "normal", + "dasllama_version" : "15", + "noise" : "ok", + "binary" : "D:/Work/daScript-iquant/bin/Release/daslang.exe", + "arch" : "x86_64", + "adopted_profile" : "x86-avx2", + "validation_max_drift_pct" : "1.84" + }, + "race" : { + "add_inplace" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 399.875, + "best_us" : 398.75 + }, + "u2" : { + "med_us" : 391.5625, + "best_us" : 391.21875 + }, + "vec8" : { + "med_us" : 399.9375, + "best_us" : 399.28125 + }, + "vec8_u2" : { + "med_us" : 391.5, + "best_us" : 390.4375 + }, + "vec16" : { + "med_us" : 395.84375, + "best_us" : 394.59375 + }, + "vec16_u2" : { + "med_us" : 393.375, + "best_us" : 392.65625 + }, + "vec4_u4" : { + "med_us" : 758.7692307692307, + "best_us" : 758.6923076923077 + } + } + }, + "cvt_f32_to_f16" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 522.53125, + "best_us" : 521.09375 + }, + "u2" : { + "med_us" : 510.625, + "best_us" : 507.375 + }, + "vec8" : { + "med_us" : 521.71875, + "best_us" : 520.875 + }, + "vec8_u2" : { + "med_us" : 509.5625, + "best_us" : 508.46875 + }, + "vec16" : { + "med_us" : 525.5625, + "best_us" : 521.3125 + }, + "vec16_u2" : { + "med_us" : 523.875, + "best_us" : 522.09375 + }, + "vec4_u4" : { + "med_us" : 646.1935483870968, + "best_us" : 643.5806451612904 + } + } + }, + "rope_scaled_neox_tab" : { + "winner" : "plain", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 19.567739672431937, + "rows" : { + "plain" : { + "med_us" : 465.8571428571429, + "best_us" : 463.7142857142857 + }, + "u2" : { + "med_us" : 579, + "best_us" : 577.84 + }, + "vec8" : { + "med_us" : 466.609375, + "best_us" : 463.75 + }, + "vec8_u2" : { + "med_us" : 579.359375, + "best_us" : 577.46875 + }, + "vec16" : { + "med_us" : 465.125, + "best_us" : 464.21875 + }, + "vec16_u2" : { + "med_us" : 562.75, + "best_us" : 562.125 + }, + "vec4_u4" : { + "med_us" : 971.2, + "best_us" : 969.7 + } + } + }, + "q51q8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "streamed_us" : 7818 + }, + "mr8" : { + "streamed_us" : 6911 + }, + "dot_maddubs_width256_mr8" : { + "streamed_us" : 1117 + }, + "dot_vpdpbusd_width256_mr8" : { + "streamed_us" : 6998 + }, + "dot_vpdpbusd_width512_mr16" : { + "streamed_us" : 6991 + }, + "dot_vpdpbusd_width256_mr16" : { + "streamed_us" : 7058 + } + } + }, + "softmax" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 5997.666666666667, + "best_us" : 5978 + }, + "u2" : { + "med_us" : 5991.666666666667, + "best_us" : 5955.666666666667 + }, + "vec8" : { + "med_us" : 6012, + "best_us" : 5953.666666666667 + }, + "vec8_u2" : { + "med_us" : 5992.333333333333, + "best_us" : 5946 + }, + "vec16" : { + "med_us" : 5980, + "best_us" : 5961.666666666667 + }, + "vec16_u2" : { + "med_us" : 6008.333333333333, + "best_us" : 5949.666666666667 + }, + "vec4_u4" : { + "med_us" : 6243.666666666667, + "best_us" : 6217.666666666667 + } + } + }, + "mul_inplace" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 400.21875, + "best_us" : 399.8125 + }, + "u2" : { + "med_us" : 390.28125, + "best_us" : 389.71875 + }, + "vec8" : { + "med_us" : 401.5625, + "best_us" : 400.75 + }, + "vec8_u2" : { + "med_us" : 390.40625, + "best_us" : 390.3125 + }, + "vec16" : { + "med_us" : 390.5, + "best_us" : 389.6875 + }, + "vec16_u2" : { + "med_us" : 389.65625, + "best_us" : 389.25 + }, + "vec4_u4" : { + "med_us" : 758.8461538461538, + "best_us" : 758.7692307692307 + } + } + }, + "quantize_q8_0_bs_into_ptr" : { + "winner" : "vec16_u2", + "fallback" : "plain", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 23.482528870081758, + "rows" : { + "plain" : { + "med_us" : 7266.75, + "best_us" : 7251 + }, + "u2" : { + "med_us" : 6704.5, + "best_us" : 6670.5 + }, + "vec8" : { + "med_us" : 7257.5, + "best_us" : 7248 + }, + "vec8_u2" : { + "med_us" : 6703, + "best_us" : 6668 + }, + "vec16" : { + "med_us" : 7273.5, + "best_us" : 7255 + }, + "vec16_u2" : { + "med_us" : 5560.333333333334, + "best_us" : 5536.666666666667 + }, + "vec4_u4" : { + "med_us" : 5583.666666666666, + "best_us" : 5554 + } + } + }, + "iq2xsq8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 1916068, + "gemv_us" : 13222, + "hot_us" : 826.9375 + }, + "mr8" : { + "tile_us" : 1876641, + "gemv_us" : 13233, + "hot_us" : 819.4375 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 109815, + "gemv_us" : 1306, + "hot_us" : 76.8125 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 1855617, + "gemv_us" : 13290, + "hot_us" : 811.0625 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 2033347, + "gemv_us" : 13260, + "hot_us" : 809.75 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 2068716, + "gemv_us" : 13451, + "hot_us" : 841.1875 + }, + "dot_vpdpbusd_width512_mr16_grid_vbmi" : { + "tile_us" : 1861400, + "gemv_us" : 13214, + "hot_us" : 830 + } + } + }, + "dot_q8kv" : { + "winner" : "vec4_u4", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 13.724232888354049, + "rows" : { + "plain" : { + "med_us" : 4280.5, + "best_us" : 4275.5 + }, + "u2" : { + "med_us" : 4608.75, + "best_us" : 4598.5 + }, + "vec8" : { + "med_us" : 4254.25, + "best_us" : 4252.5 + }, + "vec8_u2" : { + "med_us" : 991.45, + "best_us" : 985.9 + }, + "vec16" : { + "med_us" : 4278.25, + "best_us" : 4248.75 + }, + "vec16_u2" : { + "med_us" : 1646.0833333333333, + "best_us" : 1644.9166666666667 + }, + "vec4_u4" : { + "med_us" : 854.1739130434783, + "best_us" : 848.5652173913044 + } + } + }, + "dot_q8q8" : { + "winner" : "vec16", + "fallback" : "vec16", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 2596.4285714285716, + "best_us" : 2553 + }, + "u2" : { + "med_us" : 2756, + "best_us" : 2707.1428571428573 + }, + "vec8" : { + "med_us" : 2574.4285714285716, + "best_us" : 2566.285714285714 + }, + "vec8_u2" : { + "med_us" : 2727, + "best_us" : 2707.4285714285716 + }, + "vec16" : { + "med_us" : 2585.8571428571427, + "best_us" : 2558.5714285714284 + }, + "vec16_u2" : { + "med_us" : 2597.285714285714, + "best_us" : 2573.4285714285716 + }, + "vec4_u4" : { + "med_us" : 4755.75, + "best_us" : 4721.75 + } + } + }, + "iq4xsq8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 1233062, + "gemv_us" : 4757, + "hot_us" : 302.5 + }, + "mr8" : { + "tile_us" : 1244110, + "gemv_us" : 4908, + "hot_us" : 294.5625 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 102132, + "gemv_us" : 614, + "hot_us" : 25.6875 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 1238365, + "gemv_us" : 4753, + "hot_us" : 295.625 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 1237794, + "gemv_us" : 4753, + "hot_us" : 303.25 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 1241299, + "gemv_us" : 4774, + "hot_us" : 310.5 + } + } + }, + "iq4nlq8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 1189439, + "gemv_us" : 4518, + "hot_us" : 279.9375 + }, + "mr8" : { + "tile_us" : 1167184, + "gemv_us" : 4760, + "hot_us" : 282.8125 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 86201, + "gemv_us" : 633, + "hot_us" : 26.1875 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 1174610, + "gemv_us" : 4579, + "hot_us" : 284 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 1172480, + "gemv_us" : 4755, + "hot_us" : 283 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 1192175, + "gemv_us" : 4842, + "hot_us" : 280.875 + } + } + }, + "quantize_q8kv_row" : { + "winner" : "plain", + "fallback" : "plain", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 3362.2, + "best_us" : 3361.6 + }, + "u2" : { + "med_us" : 3434.4, + "best_us" : 3430.8 + }, + "vec8" : { + "med_us" : 3363.4, + "best_us" : 3359.8 + }, + "vec8_u2" : { + "med_us" : 3434, + "best_us" : 3430.6 + }, + "vec16" : { + "med_us" : 3362.2, + "best_us" : 3359.8 + }, + "vec16_u2" : { + "med_us" : 3471, + "best_us" : 3458.4 + }, + "vec4_u4" : { + "med_us" : 3883.2, + "best_us" : 3877.8 + } + } + }, + "axpy_f16" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 397.375, + "best_us" : 397.09375 + }, + "u2" : { + "med_us" : 386.6875, + "best_us" : 386 + }, + "vec8" : { + "med_us" : 399.40625, + "best_us" : 398.53125 + }, + "vec8_u2" : { + "med_us" : 385.875, + "best_us" : 385.625 + }, + "vec16" : { + "med_us" : 384.5, + "best_us" : 383.625 + }, + "vec16_u2" : { + "med_us" : 385.75, + "best_us" : 384.9375 + }, + "vec4_u4" : { + "med_us" : 760.2307692307693, + "best_us" : 759.7307692307693 + } + } + }, + "q40q8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 1261777, + "gemv_us" : 4842, + "hot_us" : 310.1875 + }, + "mr8" : { + "tile_us" : 1266068, + "gemv_us" : 4851, + "hot_us" : 303.25 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 55268, + "gemv_us" : 574, + "hot_us" : 18.75 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 1269518, + "gemv_us" : 4832, + "hot_us" : 302.0625 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 1257965, + "gemv_us" : 4885, + "hot_us" : 302.375 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 1271116, + "gemv_us" : 4953, + "hot_us" : 301.75 + } + } + }, + "axpy" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 400.59375, + "best_us" : 399.71875 + }, + "u2" : { + "med_us" : 394.1875, + "best_us" : 390.40625 + }, + "vec8" : { + "med_us" : 401.15625, + "best_us" : 400.53125 + }, + "vec8_u2" : { + "med_us" : 392.40625, + "best_us" : 391.75 + }, + "vec16" : { + "med_us" : 392, + "best_us" : 391.40625 + }, + "vec16_u2" : { + "med_us" : 392.90625, + "best_us" : 391.5 + }, + "vec4_u4" : { + "med_us" : 766, + "best_us" : 764.5 + } + } + }, + "dot_q8q8kv" : { + "winner" : "vec16", + "fallback" : "vec16", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 752.8461538461538, + "best_us" : 752 + }, + "u2" : { + "med_us" : 757.4230769230769, + "best_us" : 753.3461538461538 + }, + "vec8" : { + "med_us" : 753.1153846153846, + "best_us" : 748.4615384615385 + }, + "vec8_u2" : { + "med_us" : 758.2692307692307, + "best_us" : 754.2307692307693 + }, + "vec16" : { + "med_us" : 753.8461538461538, + "best_us" : 752.9615384615385 + }, + "vec16_u2" : { + "med_us" : 752.3076923076923, + "best_us" : 751 + }, + "vec4_u4" : { + "med_us" : 1132.0588235294117, + "best_us" : 1131.5294117647059 + } + } + }, + "dot_mx4q8" : { + "winner" : "u2", + "fallback" : "u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 2476.875, + "best_us" : 2469.25 + }, + "u2" : { + "med_us" : 2445.125, + "best_us" : 2439.375 + }, + "vec8" : { + "med_us" : 2460.625, + "best_us" : 2458.5 + }, + "vec8_u2" : { + "med_us" : 2443.375, + "best_us" : 2439.75 + }, + "vec16" : { + "med_us" : 2470.875, + "best_us" : 2468.25 + }, + "vec16_u2" : { + "med_us" : 2449.5, + "best_us" : 2443.125 + }, + "vec4_u4" : { + "med_us" : 2448.875, + "best_us" : 2438.5 + } + } + }, + "iq2xxsq8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 2012585, + "gemv_us" : 12546, + "hot_us" : 789.125 + }, + "mr8" : { + "tile_us" : 1539912, + "gemv_us" : 12583, + "hot_us" : 797.75 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 100141, + "gemv_us" : 1408, + "hot_us" : 83.3125 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 1659192, + "gemv_us" : 12802, + "hot_us" : 801.6875 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 1597520, + "gemv_us" : 12681, + "hot_us" : 791.4375 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 1495737, + "gemv_us" : 12574, + "hot_us" : 774.5 + }, + "dot_vpdpbusd_width512_mr16_grid_vbmi" : { + "tile_us" : 1653456, + "gemv_us" : 12798, + "hot_us" : 798.0625 + } + } + }, + "iq2sq8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 1984356, + "gemv_us" : 13496, + "hot_us" : 838.8125 + }, + "mr8" : { + "tile_us" : 1842888, + "gemv_us" : 13170, + "hot_us" : 827.3125 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 108514, + "gemv_us" : 1472, + "hot_us" : 81.5 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 2046296, + "gemv_us" : 13259, + "hot_us" : 826.9375 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 1952808, + "gemv_us" : 13473, + "hot_us" : 827.8125 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 1862194, + "gemv_us" : 13138, + "hot_us" : 832.25 + }, + "dot_vpdpbusd_width512_mr16_grid_vbmi" : { + "tile_us" : 1864416, + "gemv_us" : 13139, + "hot_us" : 834.4375 + } + } + }, + "cvt_q8kv_to_f32" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 934.8095238095239, + "best_us" : 919.6190476190476 + }, + "u2" : { + "med_us" : 927.3809523809524, + "best_us" : 927.1428571428571 + }, + "vec8" : { + "med_us" : 930.3809523809524, + "best_us" : 924.9047619047619 + }, + "vec8_u2" : { + "med_us" : 927.4761904761905, + "best_us" : 926.5238095238095 + }, + "vec16" : { + "med_us" : 932.0952380952381, + "best_us" : 926 + }, + "vec16_u2" : { + "med_us" : 905.7727272727273, + "best_us" : 901.7727272727273 + }, + "vec4_u4" : { + "med_us" : 929.1904761904761, + "best_us" : 926.4285714285714 + } + } + }, + "axpy_q8kv" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 564.8076923076923, + "best_us" : 561.2692307692307 + }, + "u2" : { + "med_us" : 529.6875, + "best_us" : 529.4375 + }, + "vec8" : { + "med_us" : 566.46875, + "best_us" : 564.65625 + }, + "vec8_u2" : { + "med_us" : 528.21875, + "best_us" : 527.5625 + }, + "vec16" : { + "med_us" : 564.40625, + "best_us" : 563.0625 + }, + "vec16_u2" : { + "med_us" : 529.65625, + "best_us" : 529.03125 + }, + "vec4_u4" : { + "med_us" : 877, + "best_us" : 873.5454545454545 + } + } + }, + "q8q8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8_kstep2", + "rows" : { + "reference" : { + "tile_us" : 437133, + "gemv_us" : 1788, + "hot_us" : 107.0625 + }, + "dot_maddubs_width256_mr8_kstep2" : { + "tile_us" : 81160, + "gemv_us" : 929, + "hot_us" : 23.4375 + } + } + }, + "quantize_q8_0_into_ptr" : { + "winner" : "plain", + "fallback" : "plain", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 7244.5, + "best_us" : 7222 + }, + "u2" : { + "med_us" : 7409, + "best_us" : 7383.5 + }, + "vec8" : { + "med_us" : 7235, + "best_us" : 7217.5 + }, + "vec8_u2" : { + "med_us" : 7391.5, + "best_us" : 7384.5 + }, + "vec16" : { + "med_us" : 7235.5, + "best_us" : 7217.5 + }, + "vec16_u2" : { + "med_us" : 7195.5, + "best_us" : 7191 + }, + "vec4_u4" : { + "med_us" : 8207, + "best_us" : 8196 + } + } + }, + "k4q8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 995714, + "gemv_us" : 3870, + "hot_us" : 236.8125 + }, + "mr8" : { + "tile_us" : 978897, + "gemv_us" : 3793, + "hot_us" : 233.5 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 69441, + "gemv_us" : 591, + "hot_us" : 18.125 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 969086, + "gemv_us" : 3795, + "hot_us" : 236.8125 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 971692, + "gemv_us" : 3849, + "hot_us" : 240 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 972652, + "gemv_us" : 3867, + "hot_us" : 233.0625 + } + } + }, + "k5q8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 1627010, + "gemv_us" : 5790, + "hot_us" : 355.375 + }, + "mr8" : { + "tile_us" : 1614298, + "gemv_us" : 5891, + "hot_us" : 353 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 73269, + "gemv_us" : 1000, + "hot_us" : 31.125 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 1630334, + "gemv_us" : 5865, + "hot_us" : 362.1875 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 1620199, + "gemv_us" : 5873, + "hot_us" : 359.5625 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 1612619, + "gemv_us" : 5789, + "hot_us" : 367.25 + } + } + }, + "k6q8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 1625079, + "gemv_us" : 10870, + "hot_us" : 667.8125 + }, + "mr8" : { + "tile_us" : 1620931, + "gemv_us" : 10883, + "hot_us" : 678.6875 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 86478, + "gemv_us" : 1439, + "hot_us" : 34 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 1625338, + "gemv_us" : 10945, + "hot_us" : 662.5625 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 1643513, + "gemv_us" : 10963, + "hot_us" : 672.375 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 1630415, + "gemv_us" : 10967, + "hot_us" : 676.25 + } + } + }, + "k3q8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 2954411, + "gemv_us" : 11374, + "hot_us" : 722.3125 + }, + "mr8" : { + "tile_us" : 2955542, + "gemv_us" : 11392, + "hot_us" : 725.25 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 91735, + "gemv_us" : 1021, + "hot_us" : 29.6875 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 2951420, + "gemv_us" : 11663, + "hot_us" : 720 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 2960217, + "gemv_us" : 11527, + "hot_us" : 714.0625 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 2942654, + "gemv_us" : 11343, + "hot_us" : 709.5625 + } + } + }, + "k2q8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 2589665, + "gemv_us" : 10319, + "hot_us" : 635.6875 + }, + "mr8" : { + "tile_us" : 2587899, + "gemv_us" : 9857, + "hot_us" : 637.25 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 92556, + "gemv_us" : 402, + "hot_us" : 20.4375 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 2587630, + "gemv_us" : 9864, + "hot_us" : 634.25 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 2585485, + "gemv_us" : 10112, + "hot_us" : 631.6875 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 2586322, + "gemv_us" : 10062, + "hot_us" : 627.5625 + } + } + }, + "gemm_f32_uk_4x16" : { + "winner" : "plain", + "fallback" : "u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 14.259316147520027, + "rows" : { + "plain" : { + "med_us" : 1663.4499999999998, + "best_us" : 1659.2 + }, + "u2" : { + "med_us" : 1944.25, + "best_us" : 1932.5 + }, + "vec8" : { + "med_us" : 1663.090909090909, + "best_us" : 1657.7272727272727 + }, + "vec8_u2" : { + "med_us" : 1938.1, + "best_us" : 1935.2 + }, + "vec16" : { + "med_us" : 1665.5833333333333, + "best_us" : 1661.1666666666667 + }, + "vec16_u2" : { + "med_us" : 1939.9, + "best_us" : 1935.1 + }, + "vec4_u4" : { + "med_us" : 1913.3, + "best_us" : 1908.8 + } + } + }, + "dot_f16" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 21918, + "best_us" : 21865 + }, + "u2" : { + "med_us" : 15750, + "best_us" : 15736 + }, + "vec8" : { + "med_us" : 310.875, + "best_us" : 308.46875 + }, + "vec8_u2" : { + "med_us" : 310.96875, + "best_us" : 310.15625 + }, + "vec16" : { + "med_us" : 319.78125, + "best_us" : 319.34375 + }, + "vec16_u2" : { + "med_us" : 319.21875, + "best_us" : 318 + }, + "vec4_u4" : { + "med_us" : 618.09375, + "best_us" : 616.78125 + } + } + }, + "cvt_f16_to_f32" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 269.375, + "best_us" : 268.53125 + }, + "u2" : { + "med_us" : 259, + "best_us" : 257.9375 + }, + "vec8" : { + "med_us" : 270.90625, + "best_us" : 269.59375 + }, + "vec8_u2" : { + "med_us" : 259.28125, + "best_us" : 258.875 + }, + "vec16" : { + "med_us" : 258.46875, + "best_us" : 258.28125 + }, + "vec16_u2" : { + "med_us" : 258.09375, + "best_us" : 256.90625 + }, + "vec4_u4" : { + "med_us" : 507.8125, + "best_us" : 507.15625 + } + } + }, + "dot" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 40775, + "best_us" : 40694 + }, + "u2" : { + "med_us" : 40502, + "best_us" : 40310 + }, + "vec8" : { + "med_us" : 6223.666666666667, + "best_us" : 6053.666666666667 + }, + "vec8_u2" : { + "med_us" : 6194, + "best_us" : 6077 + }, + "vec16" : { + "med_us" : 6290, + "best_us" : 6198.666666666667 + }, + "vec16_u2" : { + "med_us" : 6218.666666666667, + "best_us" : 6171.666666666667 + }, + "vec4_u4" : { + "med_us" : 7092.5, + "best_us" : 7075.5 + } + } + }, + "iq3xxsq8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 1966720, + "gemv_us" : 12753, + "hot_us" : 798.375 + }, + "mr8" : { + "tile_us" : 1484843, + "gemv_us" : 12879, + "hot_us" : 798.75 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 99394, + "gemv_us" : 1908, + "hot_us" : 118.3125 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 1629518, + "gemv_us" : 13095, + "hot_us" : 809.5625 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 1561551, + "gemv_us" : 12673, + "hot_us" : 788.5625 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 1557888, + "gemv_us" : 12867, + "hot_us" : 799.375 + }, + "dot_vpdpbusd_width512_mr16_grid_vbmi" : { + "tile_us" : 1505675, + "gemv_us" : 12873, + "hot_us" : 797.5 + } + } + }, + "iq3sq8_tile_gen" : { + "winner" : "dot_maddubs_width256_mr8", + "rows" : { + "reference" : { + "tile_us" : 1829853, + "gemv_us" : 12923, + "hot_us" : 823.0625 + }, + "mr8" : { + "tile_us" : 1484953, + "gemv_us" : 13179, + "hot_us" : 826.4375 + }, + "dot_maddubs_width256_mr8" : { + "tile_us" : 101724, + "gemv_us" : 2166, + "hot_us" : 133.375 + }, + "dot_vpdpbusd_width256_mr8" : { + "tile_us" : 1540893, + "gemv_us" : 13304, + "hot_us" : 831.25 + }, + "dot_vpdpbusd_width512_mr16" : { + "tile_us" : 1507446, + "gemv_us" : 13119, + "hot_us" : 819.125 + }, + "dot_vpdpbusd_width256_mr16" : { + "tile_us" : 1488925, + "gemv_us" : 13303, + "hot_us" : 825.625 + }, + "dot_vpdpbusd_width512_mr16_grid_vbmi" : { + "tile_us" : 1491434, + "gemv_us" : 13384, + "hot_us" : 833.875 + } + } + }, + "scale_inplace" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 269.4375, + "best_us" : 269.09375 + }, + "u2" : { + "med_us" : 260.28125, + "best_us" : 259.75 + }, + "vec8" : { + "med_us" : 269.3125, + "best_us" : 268.71875 + }, + "vec8_u2" : { + "med_us" : 259.4375, + "best_us" : 258.75 + }, + "vec16" : { + "med_us" : 260.4375, + "best_us" : 260.125 + }, + "vec16_u2" : { + "med_us" : 268.84375, + "best_us" : 266.09375 + }, + "vec4_u4" : { + "med_us" : 507.53125, + "best_us" : 506.59375 + } + } + }, + "dot_q4" : { + "winner" : "vec8_u2", + "fallback" : "vec4_u4", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 13.72621386624403, + "rows" : { + "plain" : { + "med_us" : 3347.2, + "best_us" : 3346.4 + }, + "u2" : { + "med_us" : 3733.2, + "best_us" : 3725 + }, + "vec8" : { + "med_us" : 3359.4, + "best_us" : 3349.4 + }, + "vec8_u2" : { + "med_us" : 1170.75, + "best_us" : 1163.75 + }, + "vec16" : { + "med_us" : 3347.6, + "best_us" : 3346.4 + }, + "vec16_u2" : { + "med_us" : 1834.4, + "best_us" : 1831.1 + }, + "vec4_u4" : { + "med_us" : 1358.0714285714287, + "best_us" : 1349.7857142857142 + } + } + }, + "copy_floats" : { + "winner" : "vec8_u2", + "fallback" : "vec8_u2", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 0, + "rows" : { + "plain" : { + "med_us" : 280.40625, + "best_us" : 279.9375 + }, + "u2" : { + "med_us" : 280.125, + "best_us" : 278.625 + }, + "vec8" : { + "med_us" : 282.90625, + "best_us" : 281.875 + }, + "vec8_u2" : { + "med_us" : 281.15625, + "best_us" : 280.125 + }, + "vec16" : { + "med_us" : 279.3125, + "best_us" : 278.625 + }, + "vec16_u2" : { + "med_us" : 280.3125, + "best_us" : 279.9375 + }, + "vec4_u4" : { + "med_us" : 282.28125, + "best_us" : 282.15625 + } + } + }, + "rmsnorm" : { + "winner" : "vec16_u2", + "fallback" : "plain", + "floor_pct" : 0.29078205921905087, + "margin_pct" : 87.33660130718954, + "rows" : { + "plain" : { + "med_us" : 6732, + "best_us" : 6616.5 + }, + "u2" : { + "med_us" : 6560, + "best_us" : 6553.5 + }, + "vec8" : { + "med_us" : 859.3235294117646, + "best_us" : 840.0588235294117 + }, + "vec8_u2" : { + "med_us" : 848.45, + "best_us" : 841.1 + }, + "vec16" : { + "med_us" : 844.0227272727273, + "best_us" : 836.7727272727273 + }, + "vec16_u2" : { + "med_us" : 852.5, + "best_us" : 838.5652173913044 + }, + "vec4_u4" : { + "med_us" : 1206.25, + "best_us" : 1204.25 + } + } + } + }, + "runtime" : { + "jobque_spin_us" : 30000, + "target_chunk_work" : 1, + "norm_par_threshold" : 256000, + "gemv_lane_cap" : 0, + "matmul_min_chunk_rows" : 16, + "batch_grid_2d" : 0, + "metal_devw_small_panel_mb" : 32, + "team_rank_gate" : -1, + "kv_store_par_threshold" : 256000, + "q8_l2_budget" : 4194304, + "rope_par_threshold" : 100000, + "jobque_join_poll" : 50, + "q8_token_block" : 128, + "requant_par_threshold" : 256000, + "metal_tensor" : "", + "matmul_min_chunk_rows_gemv" : 64, + "attn_par_threshold" : 100000, + "batch_lane_cap" : 0, + "metal_cvt_min_rows" : 256, + "metal_tall_floor" : 64, + "act_par_threshold" : 100000, + "threads" : 16, + "dispatch_worker_limit" : 0, + "q8_batch_chunks_per_job" : 4, + "q8_chunks_per_job" : 2, + "q4_chunks_per_job" : 4 + } +} \ No newline at end of file diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index c14a96a7e8..f36214d1f2 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -1021,5 +1021,21 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. - 2026-09-01: Intel v6 (rebased, seat race at the engine shape but n=2048): every row >= 0.97 except k3 0.89 - the tiled fixture's row length flipped the k3 seat to 256_mr16 again; the seat fixture is now built at n=14336 (29dc0b35f). Intel v7 and zen4 v7 re-mints running. +- 2026-09-01: the board rig's parity pregate "flake" was a spawn defect, not the engine: `popen_argv` + builds a CreateProcess command line with a NULL application name, and Windows refuses a RELATIVE + argv[0] spelled with forward slashes (`modules/.../dasllama-bench.exe` -> spawn fails, rc -1, no + child) while `./x.exe`, `../x.exe`, absolute and backslash spellings all resolve. The rig spawned + exactly that spelling; the same child run by hand passed 40/40. Fixed at the source - the spawn + respells argv[0] natively on Windows (every `popen_argv` / `spawn_argv` / `popen_argv_pipe` caller), + `tests/fio/popen_argv.das` spawns the offending shape, `run_and_stream` now logs a failed spawn + instead of "pregate output above" with nothing above, and the caller-side backslash workaround + left `daslib/fio.das` and the language skill. +- 2026-09-01: the zen2 board row for gemma-4-12B Q4_K_M re-minted on the fixed rig (`records/zen2.json`, + 16 threads, reference unpinned as the 08-04 sweep ran it): das pp512 66.4 +-1.7 / tg128 6.74 +-0.05 + against the clean-cpu reference 56.4 / 6.45 = 1.18x / 1.04x; the 08-04 row read 70.2 / 7.30 against + 59.1 / 6.80 = 1.19x / 1.07x. Both engines sit ~5% under August on this box (a different reference + build too, 6c84c7d5d against ebd048f); the ratio held. Pinning the reference made no difference + today (56.8 pinned, 56.4 bare). The das side ran the adopted x86-avx2 class profile (sidecar + snapshot `zen2.tune.3fe069fc370e.json`), the 08-04 row a paranoid box mint. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 09a1702374b9106af2fea3323b23ec7e765c471b Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 20:21:28 -0700 Subject: [PATCH 120/123] spawn fix, style round: the three caller-side backslash normalizes go, the fio doc claims only what holds off Windows, the test names its launch dir das_version's probe-verified normalize and the two dastest twins were the workaround the respelling retires; "either slash style on every platform" over-claimed for POSIX, where a backslash is not a separator; the failed-spawn log takes the file's user-facing prefix. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014sNMfrttRvpmQypECWoV3x --- daslib/fio.das | 3 +-- .../dasLLAMA/performance/profile_common.das | 10 ++-------- skills/daslang/references/files-and-paths.md | 4 ++-- tests/dastest/test_dasfmt_exclude_mask.das | 5 +---- tests/dastest/test_preflight_config.das | 5 +---- tests/fio/popen_argv.das | 19 +++++++++---------- 6 files changed, 16 insertions(+), 30 deletions(-) diff --git a/daslib/fio.das b/daslib/fio.das index db1d3311e2..2b3e032cac 100644 --- a/daslib/fio.das +++ b/daslib/fio.das @@ -692,8 +692,7 @@ def rmdir_rec_result(path : string) : fs_result_bool { def run_and_capture(args : array; var output : string&; timeout_sec : float = 0.0) : int { //! Run an external command and capture its stdout+stderr (merged into one pipe by the underlying ``popen_argv``). Returns the process exit code; - //! -1 means the spawn itself failed. No shell is involved; the program path (``args[0]``) may use either slash style on every - //! platform (``popen_argv`` respells it natively before the Windows CreateProcess call, which rejects ``bin/daslang``-style forward-slash relative paths). + //! -1 means the spawn itself failed. No shell is involved, and a forward-slash ``args[0]`` spawns on every platform (``popen_argv`` hands Windows the backslash spelling). var captured : string let exit_code = unsafe(popen_argv(args, timeout_sec, $(f) { if (f != null) { diff --git a/modules/dasLLAMA/performance/profile_common.das b/modules/dasLLAMA/performance/profile_common.das index fe6e52e8d4..375342d197 100644 --- a/modules/dasLLAMA/performance/profile_common.das +++ b/modules/dasLLAMA/performance/profile_common.das @@ -315,7 +315,7 @@ def run_and_stream(args : array; var tail_out : string&; keep_tail : int var acc = "" let rc = unsafe(popen_argv(args, 0.0, $(f) { if (f == null) { - to_log(LOG_ERROR, "run_and_stream: could not spawn {args[0]} (no child, no output)\n") + to_log(LOG_ERROR, "dasLLAMA bench: could not spawn {args[0]} (no child, no output)\n") return } while (true) { @@ -411,13 +411,7 @@ def run_capture(cmd : string; first_line : bool) : string { // The running daslang's version, via the box's binary (DASLANG_BIN, default bin/daslang from the // repo-root cwd these tools run in). Takes the last token of the first --version line ("0.6.3"). def das_version() : string { - // shell-free spawn (fio's run_and_capture → popen_argv), so no cmd and no quote/option traps; - // CreateProcess still wants backslashes in the program path (probe-verified: `bin/daslang` - // fails to spawn, `bin\daslang` runs), hence the normalize - var bin = g_env_prof.daslang_bin - if (is_windows()) { - bin = replace(bin, "/", "\\") - } + let bin = g_env_prof.daslang_bin var out = "" if (run_and_capture([bin, "--version"], out) != 0) { return "unknown" diff --git a/skills/daslang/references/files-and-paths.md b/skills/daslang/references/files-and-paths.md index 27592527eb..db2bcea301 100644 --- a/skills/daslang/references/files-and-paths.md +++ b/skills/daslang/references/files-and-paths.md @@ -63,8 +63,8 @@ fopen(path, "rb") $(f) { concept-assert raw POD. The array forms **panic above 2 GiB**; `long_fread` / `long_fwrite` return `int64` and have no cap. - `run_and_capture(args, var output, timeout_sec = 0.0) : int` runs a child with no shell, capturing - merged stdout+stderr. `args[0]` may use either slash style on every host - the spawn respells it - natively for Windows, whose CreateProcess rejects a forward-slash relative path. + merged stdout+stderr. A forward-slash `args[0]` spawns on every host - the spawn hands Windows + the backslash spelling its CreateProcess wants. ## Mutating operations and their three error forms diff --git a/tests/dastest/test_dasfmt_exclude_mask.das b/tests/dastest/test_dasfmt_exclude_mask.das index eadda48187..567a6d7ef0 100644 --- a/tests/dastest/test_dasfmt_exclude_mask.das +++ b/tests/dastest/test_dasfmt_exclude_mask.das @@ -7,10 +7,7 @@ require daslib/fio [test] def test_dasfmt_exclude_mask(t : T?) { - var exe = get_command_line_arguments()[0] - if (get_platform_name() == "windows") { - exe = exe |> replace("/", "\\") - } + let exe = get_command_line_arguments()[0] let tool = path_join(get_das_root(), "utils/das-fmt/dasfmt.das") var terr : string let tmp = temp_directory(terr) diff --git a/tests/dastest/test_preflight_config.das b/tests/dastest/test_preflight_config.das index ee64a62ef1..e677bf3133 100644 --- a/tests/dastest/test_preflight_config.das +++ b/tests/dastest/test_preflight_config.das @@ -60,10 +60,7 @@ def test_preflight_config(tt : T?) { } tt |> run("--show-help lists the rail-skip flag the skills hardcode") <| @(t : T?) { - var exe = get_command_line_arguments()[0] - if (get_platform_name() == "windows") { - exe = exe |> replace("/", "\\") - } + let exe = get_command_line_arguments()[0] let tool = path_join(get_das_root(), "utils/internal/preflight/main.das") var out : string let rc = run_and_capture([exe, tool, "--", "--show-help"], out, 120.0) diff --git a/tests/fio/popen_argv.das b/tests/fio/popen_argv.das index 2a6bab7595..97f7b79791 100644 --- a/tests/fio/popen_argv.das +++ b/tests/fio/popen_argv.das @@ -114,21 +114,20 @@ def test_popen_argv_arg_with_embedded_quote(t : T?) { [test] def test_popen_argv_relative_forward_slash_exe(t : T?) { - t |> run("a relative argv[0] spelled with forward slashes spawns") @(t : T?) { - // "Release/daslang.exe" from bin/ - no leading ./, forward slash: the one spelling - // Windows CreateProcess refuses unless the spawn respells it natively + t |> run("a relative argv[0] with forward slashes and no leading ./ spawns") @(t : T?) { let exe = get_full_file_name(das_exe()) - let parent = dir_name(exe) - let rel = "{base_name(parent)}/{base_name(exe)}" - let saved = getcwd() - if (!chdir(dir_name(parent))) { - t |> failure("could not chdir to {dir_name(parent)}") + let exe_dir = dir_name(exe) + let rel = "{base_name(exe_dir)}/{base_name(exe)}" + let launch_dir = dir_name(exe_dir) + let saved_cwd = getcwd() + if (!chdir(launch_dir)) { + t |> failure("could not chdir to {launch_dir}") return } var output : string let rc = run_argv([rel, "--help"], output) - t |> success(chdir(saved), "could not restore the working directory {saved}") - t |> equal(rc, 0, "spawning {rel} from {dir_name(parent)} failed") + t |> success(chdir(saved_cwd), "could not restore the working directory {saved_cwd}") + t |> equal(rc, 0, "spawning {rel} from {launch_dir} failed") t |> success(find(output, "daslang version") >= 0, "expected 'daslang version' in output: {output}") } From fb76369437ce05f21aefe980afc1d81c91fc0ab9 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 20:47:15 -0700 Subject: [PATCH 121/123] preflight round: the lattice test's grid words come from one helper, the salt's length escape rides its complexity one, and the zen2 12B row backs out until Parsec is stopped Lint: the five move-assigns into a fresh array become grid_words(kq) with move returns, the alphabet is const, the +-1 ternary casts once; jit_env_salt is one flat fold per emission input at 81 lines, so the STYLE038 escape joins the STYLE037 one with the same reason. The records gate rejects a run stamped with the parsecd daemon, which runs as a service on zen2, so the re-measured row (pp 1.18x, tg 1.04x) stays in the plan ledger with its instrument and the store keeps the 08-04 row until a rig run with Parsec stopped. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014sNMfrttRvpmQypECWoV3x --- .../dasLLAMA/performance/records/zen2.json | 51 +- .../records/zen2.tune.3fe069fc370e.json | 1496 ----------------- modules/dasLLAMA/tests/test_kquant.das | 30 +- modules/dasLLVM/daslib/llvm_jit_run.das | 2 +- plans/kernel_parity_pass.md | 9 +- 5 files changed, 48 insertions(+), 1540 deletions(-) delete mode 100644 modules/dasLLAMA/performance/records/zen2.tune.3fe069fc370e.json diff --git a/modules/dasLLAMA/performance/records/zen2.json b/modules/dasLLAMA/performance/records/zen2.json index 0e759ec63f..2aa8b17916 100644 --- a/modules/dasLLAMA/performance/records/zen2.json +++ b/modules/dasLLAMA/performance/records/zen2.json @@ -462,40 +462,37 @@ "flavor":"tuned", "box":"zen2", "threads":16, - "date":"2026-09-01", - "cmd":"modules\\dasLLAMA\\performance\\_rig\\dasllama-bench\\dasllama-bench.exe -- -m D:/Work/llama.cpp/models/gemma-4-12B-it-Q4_K_M.gguf -p 512 -n 128 -r 5 -t 16 -o json --json-path modules/dasLLAMA/performance/records/_cell_zen2.json", + "date":"2026-08-04", + "cmd":"modules/dasLLAMA/performance/_rig/dasllama-bench/dasllama-bench.exe -- -m D:\\work\\llama.cpp\\models\\gemma-4-12B-it-Q4_K_M.gguf -p 512 -n 128 -r 5 -t 16 -o json --json-path modules/dasLLAMA/performance/records/_cell_zen2.json", "hardware":{ "cpu":"AMD Ryzen Threadripper 3990X 64-Core Processor", "arch":"AMD64", - "os":"Microsoft Windows [Version 10.0.26200.9168]", + "os":"Microsoft Windows [Version 10.0.26200.8875]", "total_cores":128, "ram_gb":256, "gpu":"NVIDIA GeForce RTX 5060 Ti", - "ram_config":"8x32 GB @ 2666 MT/s", + "ram_config":"8×32 GB @ 2666 MT/s", "power_plan":"High performance", "smt":"on", - "remote_desktop":"parsec" + "remote_desktop":"off" }, "tests":{ "pp512":{ - "tok_s":66.41365246733001, - "stddev":1.7098919153213501 + "tok_s":70.207003056096113, + "stddev":0.25786811113357544 }, "tg128":{ - "tok_s":6.7354839007033362, - "stddev":0.045002911239862442 + "tok_s":7.301398548790532, + "stddev":0.009024646133184433 } }, "source":"official", - "sha":"6f7da32ed", - "version":"unknown", - "dasllama_version":15, - "tune":"k4q8_tile_gen=dot_maddubs_width256_mr8 (manifest); k5q8_tile_gen=dot_maddubs_width256_mr8 (manifest); k6q8_tile_gen=dot_maddubs_width256_mr8 (manifest); q40q8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq4xsq8_tile_gen=dot_maddubs_width256_mr8 (manifest); k3q8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq3sq8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq3xxsq8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq4nlq8_tile_gen=dot_maddubs_width256_mr8 (manifest); k2q8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq2sq8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq2xsq8_tile_gen=dot_maddubs_width256_mr8 (manifest); iq2xxsq8_tile_gen=dot_maddubs_width256_mr8 (manifest); q8q8_tile_gen=dot_maddubs_width256_mr8_kstep2 (manifest); q51q8_tile_gen=dot_maddubs_width256_mr8 (manifest)", - "tune_sha":"3fe069fc370ef67773722b024397886cce90777ee6859a22df23099b9c9683fb", - "noise":"ok", - "parity":"ok", + "sha":"37a2c1662", + "version":"0.6.4", + "tune":"k4q8_tile_gen=dot_maddubs_width256_mr8 (manifest); k5q8_tile_gen=dot_maddubs_width256_mr8 (manifest); k6q8_tile_gen=dot_maddubs_width256_mr8 (manifest); q40q8_tile_gen=dot_maddubs_width256_mr8 (manifest); q8q8_tile_gen=dot_maddubs_width256_mr8_kstep2 (manifest); q51q8_gemv_gen=mr8 (manifest)", + "tune_sha":"fe4d9873989abfef6cdca7eba5a000845d781cf3c4bb7e7673db5d83335e4ece", "exec_fmt":"weights k4/k6/q8 native; q8 activations; f32 scales; planar x64-gen (repacked)", - "env":"DASLLAMA_MODELS_DIR=D:/Work/llama.cpp/models/", + "env":"DASLLAMA_BOX=zen2 DASLLAMA_MODELS_DIR=D:\\work\\llama.cpp\\models\\", "files":[ { "role":"weights", @@ -511,32 +508,32 @@ "flavor":"clean-cpu", "box":"zen2", "threads":16, - "date":"2026-09-01", - "cmd":"D:/Work/llama.cpp/build-clean-cpu/bin/Release/llama-bench.exe -m D:/Work/llama.cpp/models/gemma-4-12B-it-Q4_K_M.gguf -ngl 0 -t 16 -p 512 -n 128 -r 5 -o json", + "date":"2026-08-04", + "cmd":"D:\\Work\\llama.cpp-ref-ebd048f\\build-clean-cpu\\bin\\Release\\llama-bench.exe -m D:\\work\\llama.cpp\\models\\gemma-4-12B-it-Q4_K_M.gguf -ngl 0 -t 16 -p 512 -n 128 -r 5 -o json", "hardware":{ "cpu":"AMD Ryzen Threadripper 3990X 64-Core Processor", "arch":"AMD64", - "os":"Microsoft Windows [Version 10.0.26200.9168]", + "os":"Microsoft Windows [Version 10.0.26200.8875]", "total_cores":128, "ram_gb":256, "gpu":"NVIDIA GeForce RTX 5060 Ti", - "ram_config":"8x32 GB @ 2666 MT/s", + "ram_config":"8×32 GB @ 2666 MT/s", "power_plan":"High performance", "smt":"on", - "remote_desktop":"parsec" + "remote_desktop":"off" }, "tests":{ "pp512":{ - "tok_s":56.404944999999998, - "stddev":0.141788 + "tok_s":59.069308999999997, + "stddev":0.26938099999999998 }, "tg128":{ - "tok_s":6.4491550000000002, - "stddev":0.035576999999999998 + "tok_s":6.7950109999999997, + "stddev":0.016497000000000001 } }, "source":"official", - "sha":"6c84c7d5d", + "sha":"ebd048fc5", "exec_fmt":"native gguf formats; cpu runtime layout repack", "files":[ { diff --git a/modules/dasLLAMA/performance/records/zen2.tune.3fe069fc370e.json b/modules/dasLLAMA/performance/records/zen2.tune.3fe069fc370e.json deleted file mode 100644 index 0ec1afa677..0000000000 --- a/modules/dasLLAMA/performance/records/zen2.tune.3fe069fc370e.json +++ /dev/null @@ -1,1496 +0,0 @@ -{ - "kernels" : { - "iq4xsq8_gemv_gen" : "dot_maddubs_width256_mr8", - "q40q8_gemv_gen" : "dot_maddubs_width256_mr8", - "add_inplace" : "vec8_u2", - "iq2xsq8_gemv_gen" : "dot_maddubs_width256_mr8", - "cvt_f32_to_f16" : "vec8_u2", - "iq2sq8_gemv_gen" : "dot_maddubs_width256_mr8", - "rope_scaled_neox_tab" : "plain", - "q51q8_tile_gen" : "dot_maddubs_width256_mr8", - "softmax" : "vec8_u2", - "mul_inplace" : "vec8_u2", - "quantize_q8_0_bs_into_ptr" : "vec16_u2", - "dot_q8kv" : "vec4_u4", - "iq2xsq8_tile_gen" : "dot_maddubs_width256_mr8", - "dot_q8q8" : "vec16", - "iq4xsq8_tile_gen" : "dot_maddubs_width256_mr8", - "iq4nlq8_tile_gen" : "dot_maddubs_width256_mr8", - "quantize_q8kv_row" : "plain", - "axpy_f16" : "vec8_u2", - "q40q8_tile_gen" : "dot_maddubs_width256_mr8", - "axpy_tq4kv" : "vec8_u2", - "cvt_tq4kv_to_f32" : "vec8_u2", - "axpy" : "vec8_u2", - "dot_mx4q8" : "u2", - "dot_q8q8kv" : "vec16", - "softmax_sink" : "vec8_u2", - "iq2xxsq8_tile_gen" : "dot_maddubs_width256_mr8", - "dot_q8q8_laneq4x4" : "", - "dot_bf16" : "vec8_u2", - "iq2sq8_tile_gen" : "dot_maddubs_width256_mr8", - "add_scale_inplace" : "vec8_u2", - "cvt_q8kv_to_f32" : "vec8_u2", - "axpy_q8kv" : "vec8_u2", - "dot_q8q8_f16s" : "vec16", - "iq2xxsq8_gemv_gen" : "dot_maddubs_width256_mr8", - "q8q8_tile_gen" : "dot_maddubs_width256_mr8_kstep2", - "quantize_q8_0_into_ptr" : "plain", - "k2q8_tile_gen" : "dot_maddubs_width256_mr8", - "gemm_f32_uk_4x16" : "plain", - "dot_q51e" : "vec16", - "k3q8_tile_gen" : "dot_maddubs_width256_mr8", - "k4q8_tile_gen" : "dot_maddubs_width256_mr8", - "k5q8_tile_gen" : "dot_maddubs_width256_mr8", - "k6q8_tile_gen" : "dot_maddubs_width256_mr8", - "k4q8_gemv_gen" : "dot_maddubs_width256_mr8", - "dot_f16" : "vec8_u2", - "k5q8_gemv_gen" : "dot_maddubs_width256_mr8", - "k6q8_gemv_gen" : "dot_maddubs_width256_mr8", - "cvt_f16_to_f32" : "vec8_u2", - "k3q8_gemv_gen" : "dot_maddubs_width256_mr8", - "iq4nlq8_gemv_gen" : "dot_maddubs_width256_mr8", - "k2q8_gemv_gen" : "dot_maddubs_width256_mr8", - "dot" : "vec8_u2", - "iq3xxsq8_tile_gen" : "dot_maddubs_width256_mr8", - "iq3xxsq8_gemv_gen" : "dot_maddubs_width256_mr8", - "iq3sq8_tile_gen" : "dot_maddubs_width256_mr8", - "dot_q8tq4kv" : "vec16", - "iq3sq8_gemv_gen" : "dot_maddubs_width256_mr8", - "scale_inplace" : "vec8_u2", - "dot_q4" : "vec8_u2", - "copy_floats" : "vec8_u2", - "quantize_tq4kv_row" : "plain", - "rmsnorm" : "vec16_u2" - }, - "provenance" : { - "validation" : "ok", - "noise_probes" : "start cv 0.29%; mid1 cv 0.19%; mid2 cv 0.09%; end cv 0.28%", - "platform" : "windows", - "noise_floor_cv_pct" : "0.29", - "features" : "avx2;f16c;fma;sse4.2", - "box" : "windows|x86_64||10.0.26200.9168|AMD64 Family 23 Model 49 Stepping 0, AuthenticAMD", - "engine_sha" : "6f7da32ed", - "written" : "2026-09-02T01:42:17.414Z", - "validation_demoted" : "0", - "mode" : "normal", - "dasllama_version" : "15", - "noise" : "ok", - "binary" : "D:/Work/daScript-iquant/bin/Release/daslang.exe", - "arch" : "x86_64", - "adopted_profile" : "x86-avx2", - "validation_max_drift_pct" : "1.84" - }, - "race" : { - "add_inplace" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 399.875, - "best_us" : 398.75 - }, - "u2" : { - "med_us" : 391.5625, - "best_us" : 391.21875 - }, - "vec8" : { - "med_us" : 399.9375, - "best_us" : 399.28125 - }, - "vec8_u2" : { - "med_us" : 391.5, - "best_us" : 390.4375 - }, - "vec16" : { - "med_us" : 395.84375, - "best_us" : 394.59375 - }, - "vec16_u2" : { - "med_us" : 393.375, - "best_us" : 392.65625 - }, - "vec4_u4" : { - "med_us" : 758.7692307692307, - "best_us" : 758.6923076923077 - } - } - }, - "cvt_f32_to_f16" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 522.53125, - "best_us" : 521.09375 - }, - "u2" : { - "med_us" : 510.625, - "best_us" : 507.375 - }, - "vec8" : { - "med_us" : 521.71875, - "best_us" : 520.875 - }, - "vec8_u2" : { - "med_us" : 509.5625, - "best_us" : 508.46875 - }, - "vec16" : { - "med_us" : 525.5625, - "best_us" : 521.3125 - }, - "vec16_u2" : { - "med_us" : 523.875, - "best_us" : 522.09375 - }, - "vec4_u4" : { - "med_us" : 646.1935483870968, - "best_us" : 643.5806451612904 - } - } - }, - "rope_scaled_neox_tab" : { - "winner" : "plain", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 19.567739672431937, - "rows" : { - "plain" : { - "med_us" : 465.8571428571429, - "best_us" : 463.7142857142857 - }, - "u2" : { - "med_us" : 579, - "best_us" : 577.84 - }, - "vec8" : { - "med_us" : 466.609375, - "best_us" : 463.75 - }, - "vec8_u2" : { - "med_us" : 579.359375, - "best_us" : 577.46875 - }, - "vec16" : { - "med_us" : 465.125, - "best_us" : 464.21875 - }, - "vec16_u2" : { - "med_us" : 562.75, - "best_us" : 562.125 - }, - "vec4_u4" : { - "med_us" : 971.2, - "best_us" : 969.7 - } - } - }, - "q51q8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "streamed_us" : 7818 - }, - "mr8" : { - "streamed_us" : 6911 - }, - "dot_maddubs_width256_mr8" : { - "streamed_us" : 1117 - }, - "dot_vpdpbusd_width256_mr8" : { - "streamed_us" : 6998 - }, - "dot_vpdpbusd_width512_mr16" : { - "streamed_us" : 6991 - }, - "dot_vpdpbusd_width256_mr16" : { - "streamed_us" : 7058 - } - } - }, - "softmax" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 5997.666666666667, - "best_us" : 5978 - }, - "u2" : { - "med_us" : 5991.666666666667, - "best_us" : 5955.666666666667 - }, - "vec8" : { - "med_us" : 6012, - "best_us" : 5953.666666666667 - }, - "vec8_u2" : { - "med_us" : 5992.333333333333, - "best_us" : 5946 - }, - "vec16" : { - "med_us" : 5980, - "best_us" : 5961.666666666667 - }, - "vec16_u2" : { - "med_us" : 6008.333333333333, - "best_us" : 5949.666666666667 - }, - "vec4_u4" : { - "med_us" : 6243.666666666667, - "best_us" : 6217.666666666667 - } - } - }, - "mul_inplace" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 400.21875, - "best_us" : 399.8125 - }, - "u2" : { - "med_us" : 390.28125, - "best_us" : 389.71875 - }, - "vec8" : { - "med_us" : 401.5625, - "best_us" : 400.75 - }, - "vec8_u2" : { - "med_us" : 390.40625, - "best_us" : 390.3125 - }, - "vec16" : { - "med_us" : 390.5, - "best_us" : 389.6875 - }, - "vec16_u2" : { - "med_us" : 389.65625, - "best_us" : 389.25 - }, - "vec4_u4" : { - "med_us" : 758.8461538461538, - "best_us" : 758.7692307692307 - } - } - }, - "quantize_q8_0_bs_into_ptr" : { - "winner" : "vec16_u2", - "fallback" : "plain", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 23.482528870081758, - "rows" : { - "plain" : { - "med_us" : 7266.75, - "best_us" : 7251 - }, - "u2" : { - "med_us" : 6704.5, - "best_us" : 6670.5 - }, - "vec8" : { - "med_us" : 7257.5, - "best_us" : 7248 - }, - "vec8_u2" : { - "med_us" : 6703, - "best_us" : 6668 - }, - "vec16" : { - "med_us" : 7273.5, - "best_us" : 7255 - }, - "vec16_u2" : { - "med_us" : 5560.333333333334, - "best_us" : 5536.666666666667 - }, - "vec4_u4" : { - "med_us" : 5583.666666666666, - "best_us" : 5554 - } - } - }, - "iq2xsq8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 1916068, - "gemv_us" : 13222, - "hot_us" : 826.9375 - }, - "mr8" : { - "tile_us" : 1876641, - "gemv_us" : 13233, - "hot_us" : 819.4375 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 109815, - "gemv_us" : 1306, - "hot_us" : 76.8125 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 1855617, - "gemv_us" : 13290, - "hot_us" : 811.0625 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 2033347, - "gemv_us" : 13260, - "hot_us" : 809.75 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 2068716, - "gemv_us" : 13451, - "hot_us" : 841.1875 - }, - "dot_vpdpbusd_width512_mr16_grid_vbmi" : { - "tile_us" : 1861400, - "gemv_us" : 13214, - "hot_us" : 830 - } - } - }, - "dot_q8kv" : { - "winner" : "vec4_u4", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 13.724232888354049, - "rows" : { - "plain" : { - "med_us" : 4280.5, - "best_us" : 4275.5 - }, - "u2" : { - "med_us" : 4608.75, - "best_us" : 4598.5 - }, - "vec8" : { - "med_us" : 4254.25, - "best_us" : 4252.5 - }, - "vec8_u2" : { - "med_us" : 991.45, - "best_us" : 985.9 - }, - "vec16" : { - "med_us" : 4278.25, - "best_us" : 4248.75 - }, - "vec16_u2" : { - "med_us" : 1646.0833333333333, - "best_us" : 1644.9166666666667 - }, - "vec4_u4" : { - "med_us" : 854.1739130434783, - "best_us" : 848.5652173913044 - } - } - }, - "dot_q8q8" : { - "winner" : "vec16", - "fallback" : "vec16", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 2596.4285714285716, - "best_us" : 2553 - }, - "u2" : { - "med_us" : 2756, - "best_us" : 2707.1428571428573 - }, - "vec8" : { - "med_us" : 2574.4285714285716, - "best_us" : 2566.285714285714 - }, - "vec8_u2" : { - "med_us" : 2727, - "best_us" : 2707.4285714285716 - }, - "vec16" : { - "med_us" : 2585.8571428571427, - "best_us" : 2558.5714285714284 - }, - "vec16_u2" : { - "med_us" : 2597.285714285714, - "best_us" : 2573.4285714285716 - }, - "vec4_u4" : { - "med_us" : 4755.75, - "best_us" : 4721.75 - } - } - }, - "iq4xsq8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 1233062, - "gemv_us" : 4757, - "hot_us" : 302.5 - }, - "mr8" : { - "tile_us" : 1244110, - "gemv_us" : 4908, - "hot_us" : 294.5625 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 102132, - "gemv_us" : 614, - "hot_us" : 25.6875 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 1238365, - "gemv_us" : 4753, - "hot_us" : 295.625 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 1237794, - "gemv_us" : 4753, - "hot_us" : 303.25 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 1241299, - "gemv_us" : 4774, - "hot_us" : 310.5 - } - } - }, - "iq4nlq8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 1189439, - "gemv_us" : 4518, - "hot_us" : 279.9375 - }, - "mr8" : { - "tile_us" : 1167184, - "gemv_us" : 4760, - "hot_us" : 282.8125 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 86201, - "gemv_us" : 633, - "hot_us" : 26.1875 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 1174610, - "gemv_us" : 4579, - "hot_us" : 284 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 1172480, - "gemv_us" : 4755, - "hot_us" : 283 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 1192175, - "gemv_us" : 4842, - "hot_us" : 280.875 - } - } - }, - "quantize_q8kv_row" : { - "winner" : "plain", - "fallback" : "plain", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 3362.2, - "best_us" : 3361.6 - }, - "u2" : { - "med_us" : 3434.4, - "best_us" : 3430.8 - }, - "vec8" : { - "med_us" : 3363.4, - "best_us" : 3359.8 - }, - "vec8_u2" : { - "med_us" : 3434, - "best_us" : 3430.6 - }, - "vec16" : { - "med_us" : 3362.2, - "best_us" : 3359.8 - }, - "vec16_u2" : { - "med_us" : 3471, - "best_us" : 3458.4 - }, - "vec4_u4" : { - "med_us" : 3883.2, - "best_us" : 3877.8 - } - } - }, - "axpy_f16" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 397.375, - "best_us" : 397.09375 - }, - "u2" : { - "med_us" : 386.6875, - "best_us" : 386 - }, - "vec8" : { - "med_us" : 399.40625, - "best_us" : 398.53125 - }, - "vec8_u2" : { - "med_us" : 385.875, - "best_us" : 385.625 - }, - "vec16" : { - "med_us" : 384.5, - "best_us" : 383.625 - }, - "vec16_u2" : { - "med_us" : 385.75, - "best_us" : 384.9375 - }, - "vec4_u4" : { - "med_us" : 760.2307692307693, - "best_us" : 759.7307692307693 - } - } - }, - "q40q8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 1261777, - "gemv_us" : 4842, - "hot_us" : 310.1875 - }, - "mr8" : { - "tile_us" : 1266068, - "gemv_us" : 4851, - "hot_us" : 303.25 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 55268, - "gemv_us" : 574, - "hot_us" : 18.75 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 1269518, - "gemv_us" : 4832, - "hot_us" : 302.0625 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 1257965, - "gemv_us" : 4885, - "hot_us" : 302.375 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 1271116, - "gemv_us" : 4953, - "hot_us" : 301.75 - } - } - }, - "axpy" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 400.59375, - "best_us" : 399.71875 - }, - "u2" : { - "med_us" : 394.1875, - "best_us" : 390.40625 - }, - "vec8" : { - "med_us" : 401.15625, - "best_us" : 400.53125 - }, - "vec8_u2" : { - "med_us" : 392.40625, - "best_us" : 391.75 - }, - "vec16" : { - "med_us" : 392, - "best_us" : 391.40625 - }, - "vec16_u2" : { - "med_us" : 392.90625, - "best_us" : 391.5 - }, - "vec4_u4" : { - "med_us" : 766, - "best_us" : 764.5 - } - } - }, - "dot_q8q8kv" : { - "winner" : "vec16", - "fallback" : "vec16", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 752.8461538461538, - "best_us" : 752 - }, - "u2" : { - "med_us" : 757.4230769230769, - "best_us" : 753.3461538461538 - }, - "vec8" : { - "med_us" : 753.1153846153846, - "best_us" : 748.4615384615385 - }, - "vec8_u2" : { - "med_us" : 758.2692307692307, - "best_us" : 754.2307692307693 - }, - "vec16" : { - "med_us" : 753.8461538461538, - "best_us" : 752.9615384615385 - }, - "vec16_u2" : { - "med_us" : 752.3076923076923, - "best_us" : 751 - }, - "vec4_u4" : { - "med_us" : 1132.0588235294117, - "best_us" : 1131.5294117647059 - } - } - }, - "dot_mx4q8" : { - "winner" : "u2", - "fallback" : "u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 2476.875, - "best_us" : 2469.25 - }, - "u2" : { - "med_us" : 2445.125, - "best_us" : 2439.375 - }, - "vec8" : { - "med_us" : 2460.625, - "best_us" : 2458.5 - }, - "vec8_u2" : { - "med_us" : 2443.375, - "best_us" : 2439.75 - }, - "vec16" : { - "med_us" : 2470.875, - "best_us" : 2468.25 - }, - "vec16_u2" : { - "med_us" : 2449.5, - "best_us" : 2443.125 - }, - "vec4_u4" : { - "med_us" : 2448.875, - "best_us" : 2438.5 - } - } - }, - "iq2xxsq8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 2012585, - "gemv_us" : 12546, - "hot_us" : 789.125 - }, - "mr8" : { - "tile_us" : 1539912, - "gemv_us" : 12583, - "hot_us" : 797.75 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 100141, - "gemv_us" : 1408, - "hot_us" : 83.3125 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 1659192, - "gemv_us" : 12802, - "hot_us" : 801.6875 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 1597520, - "gemv_us" : 12681, - "hot_us" : 791.4375 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 1495737, - "gemv_us" : 12574, - "hot_us" : 774.5 - }, - "dot_vpdpbusd_width512_mr16_grid_vbmi" : { - "tile_us" : 1653456, - "gemv_us" : 12798, - "hot_us" : 798.0625 - } - } - }, - "iq2sq8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 1984356, - "gemv_us" : 13496, - "hot_us" : 838.8125 - }, - "mr8" : { - "tile_us" : 1842888, - "gemv_us" : 13170, - "hot_us" : 827.3125 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 108514, - "gemv_us" : 1472, - "hot_us" : 81.5 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 2046296, - "gemv_us" : 13259, - "hot_us" : 826.9375 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 1952808, - "gemv_us" : 13473, - "hot_us" : 827.8125 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 1862194, - "gemv_us" : 13138, - "hot_us" : 832.25 - }, - "dot_vpdpbusd_width512_mr16_grid_vbmi" : { - "tile_us" : 1864416, - "gemv_us" : 13139, - "hot_us" : 834.4375 - } - } - }, - "cvt_q8kv_to_f32" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 934.8095238095239, - "best_us" : 919.6190476190476 - }, - "u2" : { - "med_us" : 927.3809523809524, - "best_us" : 927.1428571428571 - }, - "vec8" : { - "med_us" : 930.3809523809524, - "best_us" : 924.9047619047619 - }, - "vec8_u2" : { - "med_us" : 927.4761904761905, - "best_us" : 926.5238095238095 - }, - "vec16" : { - "med_us" : 932.0952380952381, - "best_us" : 926 - }, - "vec16_u2" : { - "med_us" : 905.7727272727273, - "best_us" : 901.7727272727273 - }, - "vec4_u4" : { - "med_us" : 929.1904761904761, - "best_us" : 926.4285714285714 - } - } - }, - "axpy_q8kv" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 564.8076923076923, - "best_us" : 561.2692307692307 - }, - "u2" : { - "med_us" : 529.6875, - "best_us" : 529.4375 - }, - "vec8" : { - "med_us" : 566.46875, - "best_us" : 564.65625 - }, - "vec8_u2" : { - "med_us" : 528.21875, - "best_us" : 527.5625 - }, - "vec16" : { - "med_us" : 564.40625, - "best_us" : 563.0625 - }, - "vec16_u2" : { - "med_us" : 529.65625, - "best_us" : 529.03125 - }, - "vec4_u4" : { - "med_us" : 877, - "best_us" : 873.5454545454545 - } - } - }, - "q8q8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8_kstep2", - "rows" : { - "reference" : { - "tile_us" : 437133, - "gemv_us" : 1788, - "hot_us" : 107.0625 - }, - "dot_maddubs_width256_mr8_kstep2" : { - "tile_us" : 81160, - "gemv_us" : 929, - "hot_us" : 23.4375 - } - } - }, - "quantize_q8_0_into_ptr" : { - "winner" : "plain", - "fallback" : "plain", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 7244.5, - "best_us" : 7222 - }, - "u2" : { - "med_us" : 7409, - "best_us" : 7383.5 - }, - "vec8" : { - "med_us" : 7235, - "best_us" : 7217.5 - }, - "vec8_u2" : { - "med_us" : 7391.5, - "best_us" : 7384.5 - }, - "vec16" : { - "med_us" : 7235.5, - "best_us" : 7217.5 - }, - "vec16_u2" : { - "med_us" : 7195.5, - "best_us" : 7191 - }, - "vec4_u4" : { - "med_us" : 8207, - "best_us" : 8196 - } - } - }, - "k4q8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 995714, - "gemv_us" : 3870, - "hot_us" : 236.8125 - }, - "mr8" : { - "tile_us" : 978897, - "gemv_us" : 3793, - "hot_us" : 233.5 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 69441, - "gemv_us" : 591, - "hot_us" : 18.125 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 969086, - "gemv_us" : 3795, - "hot_us" : 236.8125 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 971692, - "gemv_us" : 3849, - "hot_us" : 240 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 972652, - "gemv_us" : 3867, - "hot_us" : 233.0625 - } - } - }, - "k5q8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 1627010, - "gemv_us" : 5790, - "hot_us" : 355.375 - }, - "mr8" : { - "tile_us" : 1614298, - "gemv_us" : 5891, - "hot_us" : 353 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 73269, - "gemv_us" : 1000, - "hot_us" : 31.125 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 1630334, - "gemv_us" : 5865, - "hot_us" : 362.1875 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 1620199, - "gemv_us" : 5873, - "hot_us" : 359.5625 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 1612619, - "gemv_us" : 5789, - "hot_us" : 367.25 - } - } - }, - "k6q8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 1625079, - "gemv_us" : 10870, - "hot_us" : 667.8125 - }, - "mr8" : { - "tile_us" : 1620931, - "gemv_us" : 10883, - "hot_us" : 678.6875 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 86478, - "gemv_us" : 1439, - "hot_us" : 34 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 1625338, - "gemv_us" : 10945, - "hot_us" : 662.5625 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 1643513, - "gemv_us" : 10963, - "hot_us" : 672.375 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 1630415, - "gemv_us" : 10967, - "hot_us" : 676.25 - } - } - }, - "k3q8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 2954411, - "gemv_us" : 11374, - "hot_us" : 722.3125 - }, - "mr8" : { - "tile_us" : 2955542, - "gemv_us" : 11392, - "hot_us" : 725.25 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 91735, - "gemv_us" : 1021, - "hot_us" : 29.6875 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 2951420, - "gemv_us" : 11663, - "hot_us" : 720 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 2960217, - "gemv_us" : 11527, - "hot_us" : 714.0625 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 2942654, - "gemv_us" : 11343, - "hot_us" : 709.5625 - } - } - }, - "k2q8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 2589665, - "gemv_us" : 10319, - "hot_us" : 635.6875 - }, - "mr8" : { - "tile_us" : 2587899, - "gemv_us" : 9857, - "hot_us" : 637.25 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 92556, - "gemv_us" : 402, - "hot_us" : 20.4375 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 2587630, - "gemv_us" : 9864, - "hot_us" : 634.25 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 2585485, - "gemv_us" : 10112, - "hot_us" : 631.6875 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 2586322, - "gemv_us" : 10062, - "hot_us" : 627.5625 - } - } - }, - "gemm_f32_uk_4x16" : { - "winner" : "plain", - "fallback" : "u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 14.259316147520027, - "rows" : { - "plain" : { - "med_us" : 1663.4499999999998, - "best_us" : 1659.2 - }, - "u2" : { - "med_us" : 1944.25, - "best_us" : 1932.5 - }, - "vec8" : { - "med_us" : 1663.090909090909, - "best_us" : 1657.7272727272727 - }, - "vec8_u2" : { - "med_us" : 1938.1, - "best_us" : 1935.2 - }, - "vec16" : { - "med_us" : 1665.5833333333333, - "best_us" : 1661.1666666666667 - }, - "vec16_u2" : { - "med_us" : 1939.9, - "best_us" : 1935.1 - }, - "vec4_u4" : { - "med_us" : 1913.3, - "best_us" : 1908.8 - } - } - }, - "dot_f16" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 21918, - "best_us" : 21865 - }, - "u2" : { - "med_us" : 15750, - "best_us" : 15736 - }, - "vec8" : { - "med_us" : 310.875, - "best_us" : 308.46875 - }, - "vec8_u2" : { - "med_us" : 310.96875, - "best_us" : 310.15625 - }, - "vec16" : { - "med_us" : 319.78125, - "best_us" : 319.34375 - }, - "vec16_u2" : { - "med_us" : 319.21875, - "best_us" : 318 - }, - "vec4_u4" : { - "med_us" : 618.09375, - "best_us" : 616.78125 - } - } - }, - "cvt_f16_to_f32" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 269.375, - "best_us" : 268.53125 - }, - "u2" : { - "med_us" : 259, - "best_us" : 257.9375 - }, - "vec8" : { - "med_us" : 270.90625, - "best_us" : 269.59375 - }, - "vec8_u2" : { - "med_us" : 259.28125, - "best_us" : 258.875 - }, - "vec16" : { - "med_us" : 258.46875, - "best_us" : 258.28125 - }, - "vec16_u2" : { - "med_us" : 258.09375, - "best_us" : 256.90625 - }, - "vec4_u4" : { - "med_us" : 507.8125, - "best_us" : 507.15625 - } - } - }, - "dot" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 40775, - "best_us" : 40694 - }, - "u2" : { - "med_us" : 40502, - "best_us" : 40310 - }, - "vec8" : { - "med_us" : 6223.666666666667, - "best_us" : 6053.666666666667 - }, - "vec8_u2" : { - "med_us" : 6194, - "best_us" : 6077 - }, - "vec16" : { - "med_us" : 6290, - "best_us" : 6198.666666666667 - }, - "vec16_u2" : { - "med_us" : 6218.666666666667, - "best_us" : 6171.666666666667 - }, - "vec4_u4" : { - "med_us" : 7092.5, - "best_us" : 7075.5 - } - } - }, - "iq3xxsq8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 1966720, - "gemv_us" : 12753, - "hot_us" : 798.375 - }, - "mr8" : { - "tile_us" : 1484843, - "gemv_us" : 12879, - "hot_us" : 798.75 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 99394, - "gemv_us" : 1908, - "hot_us" : 118.3125 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 1629518, - "gemv_us" : 13095, - "hot_us" : 809.5625 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 1561551, - "gemv_us" : 12673, - "hot_us" : 788.5625 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 1557888, - "gemv_us" : 12867, - "hot_us" : 799.375 - }, - "dot_vpdpbusd_width512_mr16_grid_vbmi" : { - "tile_us" : 1505675, - "gemv_us" : 12873, - "hot_us" : 797.5 - } - } - }, - "iq3sq8_tile_gen" : { - "winner" : "dot_maddubs_width256_mr8", - "rows" : { - "reference" : { - "tile_us" : 1829853, - "gemv_us" : 12923, - "hot_us" : 823.0625 - }, - "mr8" : { - "tile_us" : 1484953, - "gemv_us" : 13179, - "hot_us" : 826.4375 - }, - "dot_maddubs_width256_mr8" : { - "tile_us" : 101724, - "gemv_us" : 2166, - "hot_us" : 133.375 - }, - "dot_vpdpbusd_width256_mr8" : { - "tile_us" : 1540893, - "gemv_us" : 13304, - "hot_us" : 831.25 - }, - "dot_vpdpbusd_width512_mr16" : { - "tile_us" : 1507446, - "gemv_us" : 13119, - "hot_us" : 819.125 - }, - "dot_vpdpbusd_width256_mr16" : { - "tile_us" : 1488925, - "gemv_us" : 13303, - "hot_us" : 825.625 - }, - "dot_vpdpbusd_width512_mr16_grid_vbmi" : { - "tile_us" : 1491434, - "gemv_us" : 13384, - "hot_us" : 833.875 - } - } - }, - "scale_inplace" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 269.4375, - "best_us" : 269.09375 - }, - "u2" : { - "med_us" : 260.28125, - "best_us" : 259.75 - }, - "vec8" : { - "med_us" : 269.3125, - "best_us" : 268.71875 - }, - "vec8_u2" : { - "med_us" : 259.4375, - "best_us" : 258.75 - }, - "vec16" : { - "med_us" : 260.4375, - "best_us" : 260.125 - }, - "vec16_u2" : { - "med_us" : 268.84375, - "best_us" : 266.09375 - }, - "vec4_u4" : { - "med_us" : 507.53125, - "best_us" : 506.59375 - } - } - }, - "dot_q4" : { - "winner" : "vec8_u2", - "fallback" : "vec4_u4", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 13.72621386624403, - "rows" : { - "plain" : { - "med_us" : 3347.2, - "best_us" : 3346.4 - }, - "u2" : { - "med_us" : 3733.2, - "best_us" : 3725 - }, - "vec8" : { - "med_us" : 3359.4, - "best_us" : 3349.4 - }, - "vec8_u2" : { - "med_us" : 1170.75, - "best_us" : 1163.75 - }, - "vec16" : { - "med_us" : 3347.6, - "best_us" : 3346.4 - }, - "vec16_u2" : { - "med_us" : 1834.4, - "best_us" : 1831.1 - }, - "vec4_u4" : { - "med_us" : 1358.0714285714287, - "best_us" : 1349.7857142857142 - } - } - }, - "copy_floats" : { - "winner" : "vec8_u2", - "fallback" : "vec8_u2", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 0, - "rows" : { - "plain" : { - "med_us" : 280.40625, - "best_us" : 279.9375 - }, - "u2" : { - "med_us" : 280.125, - "best_us" : 278.625 - }, - "vec8" : { - "med_us" : 282.90625, - "best_us" : 281.875 - }, - "vec8_u2" : { - "med_us" : 281.15625, - "best_us" : 280.125 - }, - "vec16" : { - "med_us" : 279.3125, - "best_us" : 278.625 - }, - "vec16_u2" : { - "med_us" : 280.3125, - "best_us" : 279.9375 - }, - "vec4_u4" : { - "med_us" : 282.28125, - "best_us" : 282.15625 - } - } - }, - "rmsnorm" : { - "winner" : "vec16_u2", - "fallback" : "plain", - "floor_pct" : 0.29078205921905087, - "margin_pct" : 87.33660130718954, - "rows" : { - "plain" : { - "med_us" : 6732, - "best_us" : 6616.5 - }, - "u2" : { - "med_us" : 6560, - "best_us" : 6553.5 - }, - "vec8" : { - "med_us" : 859.3235294117646, - "best_us" : 840.0588235294117 - }, - "vec8_u2" : { - "med_us" : 848.45, - "best_us" : 841.1 - }, - "vec16" : { - "med_us" : 844.0227272727273, - "best_us" : 836.7727272727273 - }, - "vec16_u2" : { - "med_us" : 852.5, - "best_us" : 838.5652173913044 - }, - "vec4_u4" : { - "med_us" : 1206.25, - "best_us" : 1204.25 - } - } - } - }, - "runtime" : { - "jobque_spin_us" : 30000, - "target_chunk_work" : 1, - "norm_par_threshold" : 256000, - "gemv_lane_cap" : 0, - "matmul_min_chunk_rows" : 16, - "batch_grid_2d" : 0, - "metal_devw_small_panel_mb" : 32, - "team_rank_gate" : -1, - "kv_store_par_threshold" : 256000, - "q8_l2_budget" : 4194304, - "rope_par_threshold" : 100000, - "jobque_join_poll" : 50, - "q8_token_block" : 128, - "requant_par_threshold" : 256000, - "metal_tensor" : "", - "matmul_min_chunk_rows_gemv" : 64, - "attn_par_threshold" : 100000, - "batch_lane_cap" : 0, - "metal_cvt_min_rows" : 256, - "metal_tall_floor" : 64, - "act_par_threshold" : 100000, - "threads" : 16, - "dispatch_worker_limit" : 0, - "q8_batch_chunks_per_job" : 4, - "q8_chunks_per_job" : 2, - "q4_chunks_per_job" : 4 - } -} \ No newline at end of file diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 5e01168dcb..81da039936 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -2570,24 +2570,26 @@ def test_kq_plane_sb_metal_blob_form(t : T?) { // The VBMI lattice's tables are pure host-side data - prove them on any box, no VBMI needed: // every grid entry's packed code decodes back to the grid's own bytes through the alphabet, // and the ksigns +-1 expansion matches a scalar re-derivation. -def private vbmi_plane_gate(t : T?; kq : int) { - var alpha <- vbmi_alphabet(kq) - let iq3 = kq == 33 || kq == 34 - let bits = iq3 ? 3 : 2 - let nbytes = iq3 ? 2 : 4 - let mask = iq3 ? 7 : 3 - var words : array +def private grid_words(kq : int) : array { if (kq == 34) { - words <- [for (w in iq3xxs_grid()); w] + return <- [for (w in iq3xxs_grid()); w] } elif (kq == 33) { - words <- [for (w in iq3s_grid()); w] + return <- [for (w in iq3s_grid()); w] } elif (kq == 25) { - words <- [for (w in iq2xxs_grid2()); w] + return <- [for (w in iq2xxs_grid2()); w] } elif (kq == 24) { - words <- [for (w in iq2xs_grid2()); w] - } else { - words <- [for (w in iq2s_grid2()); w] + return <- [for (w in iq2xs_grid2()); w] } + return <- [for (w in iq2s_grid2()); w] +} + +def private vbmi_plane_gate(t : T?; kq : int) { + let alpha <- vbmi_alphabet(kq) + let iq3 = kq == 33 || kq == 34 + let bits = iq3 ? 3 : 2 + let nbytes = iq3 ? 2 : 4 + let mask = iq3 ? 7 : 3 + let words <- grid_words(kq) var bad = 0 for (word in words) { for (half in range(iq3 ? 2 : 1)) { // iq3: one word = two plane bytes (low/high pair); iq2: one word = one plane byte @@ -2619,7 +2621,7 @@ def test_vbmi_lattice_tables(t : T?) { let packed = pm1_of(sb) for (k in range(8)) { let lane = int8(int(packed >> uint64(8 * k)) & 0xFF) - let want = (sb & (1 << k)) != 0 ? int8(-1) : int8(1) + let want = int8((sb & (1 << k)) != 0 ? -1 : 1) if (lane != want) { bad++ } diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 69cb9c1274..0048c0d457 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -176,7 +176,7 @@ def private has_generated_kernel(prog : Program?; jit_all_functions : bool) : bo // The environment/config salt shared by the whole-DLL key AND the per-partition obj cache key. // ONE helper on purpose: a salt component folded into the DLL key but not the partition key // would let a config change reuse stale cached partition objects. -def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; // nolint:STYLE037 - one flat fold per emission input; a split would hide which inputs key the cache +def jit_env_salt(opt_level : int; size_level : int; emit_prologue : bool; debug_info : bool; // nolint:STYLE037,STYLE038 - one flat fold per emission input; a split would hide which inputs key the cache target_triple : string; split : bool; use_host_cpu : bool) : uint64 { var h = LLVM_JIT_CODEGEN_VERSION // host/binary identity: object format follows the triple (COFF vs ELF on one checkout), and the emitting binary bakes its Context offsets into every global access - neither is in any AST hash diff --git a/plans/kernel_parity_pass.md b/plans/kernel_parity_pass.md index f36214d1f2..2b5b591ed2 100644 --- a/plans/kernel_parity_pass.md +++ b/plans/kernel_parity_pass.md @@ -1035,7 +1035,12 @@ Vulkan grid tg (gap 3): followup_vulkan 35's levers, after gap 1 or 2 lands. against the clean-cpu reference 56.4 / 6.45 = 1.18x / 1.04x; the 08-04 row read 70.2 / 7.30 against 59.1 / 6.80 = 1.19x / 1.07x. Both engines sit ~5% under August on this box (a different reference build too, 6c84c7d5d against ebd048f); the ratio held. Pinning the reference made no difference - today (56.8 pinned, 56.4 bare). The das side ran the adopted x86-avx2 class profile (sidecar - snapshot `zen2.tune.3fe069fc370e.json`), the 08-04 row a paranoid box mint. + today (56.8 pinned, 56.4 bare). The das side ran the adopted x86-avx2 class profile (sidecar sha + 3fe069fc370e), the 08-04 row a paranoid box mint. NOT STORED: the records gate (`performance/REVIEW.das`) + rejects a run whose `hardware.remote_desktop` reads `parsec`, and the stamp fires on the parsecd + daemon's presence - it runs as a service on this box - so the row backs out of the branch and the + cell is re-minted once Parsec is stopped for the ten-minute rig window (Boris's call; the daemon is + his remote access). Instrument: `gen_bench_records.das -- -o gemma-4-12B --workload llm + --ref-no-affinity`, both walls out-of-process, direction-grade against the 08-04 row. - 2026-09-01: step 0 done - `kq_kernel_bench.das`, the reference rows, both memos, the fact base above. `test-backend-ops` built in `build-clean-cpu` and `build-vulkan` with the thread pin. From 2d7a6a3e52c6cc65c3b605a324c5de0f04da3ef1 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 21:57:27 -0700 Subject: [PATCH 122/123] the GEP scanner matches the call shape and ignores comments: a mention of the banned name next to the wrapper is not a call site Copilot's point on the rebased tip: the substring scan would fail the gate on the very comment the REVIEW rule invites. The scanner now drops a trailing // comment from each line and looks for LLVMSetIsInBounds( only. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014sNMfrttRvpmQypECWoV3x --- modules/dasLLVM/tests/llvm_gep_inbounds.das | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/dasLLVM/tests/llvm_gep_inbounds.das b/modules/dasLLVM/tests/llvm_gep_inbounds.das index 0131f29b12..349fa845fb 100644 --- a/modules/dasLLVM/tests/llvm_gep_inbounds.das +++ b/modules/dasLLVM/tests/llvm_gep_inbounds.das @@ -30,7 +30,9 @@ def private scan_dir(dir : string; var offenders : array) { let text = fread(path) var ln = 1 for (line in split(text, "\n")) { - if (line |> find("LLVMSetIsInBounds") >= 0) { + let cmt = line |> find("//") + let code = cmt >= 0 ? slice(line, 0, cmt) : line // a mention in a comment is not a call + if (code |> find("LLVMSetIsInBounds(") >= 0) { offenders |> push("{path}:{ln}") } ln++ From 23cc1a3bb5f89852f37d4cda0780ddf0e29a12dd Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Tue, 1 Sep 2026 22:43:06 -0700 Subject: [PATCH 123/123] kq_kernel_bench reads the tune mode off g_env_tune, not the environment: the registry test the linux lane runs was red since the lab landed The two get_env_variable("DAS_TUNE_MODE") reads become g_env_tune.tune_mode through llvm/daslib/llvm_env - the knob's one declared read site, as modules/dasLLAMA/REVIEW.md and tests/test_env_registry.das require. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_014sNMfrttRvpmQypECWoV3x --- modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das index 8d0df0fe00..c727d4cb70 100644 --- a/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das +++ b/modules/dasLLAMA/benchmarks/matmul/kq_kernel_bench.das @@ -5,13 +5,13 @@ options _dasllama_internal = true require daslib/clargs require daslib/enum_trait -require daslib/fio require daslib/jobque_boost require dasllama/dasllama_par require dasllama/dasllama_math_gen require dasllama/dasllama_math require dasllama/dasllama_kqformat require dasllama/dasllama_gemm_schema +require llvm/daslib/llvm_env require math require strings require daslib/strings_boost @@ -356,7 +356,7 @@ def emit_row(var rows : int&; name, tier : string; var st : RowStat; n, d, ntok, let ns_per_sb = float(best) * 1000.0 / float(d * nsb * ntok) let gbs = float(plane_bytes) / (float(best) * 1000.0) let gflops = 2.0 * float(n) * float(d) * float(ntok) / (float(best) * 1000.0) - let mode = get_env_variable("DAS_TUNE_MODE") + let mode = g_env_tune.tune_mode let perm = (st.perm == "reference" && mode != "tune" && mode != "test") ? "stamped" : st.perm // normal mode: the registry's one row IS the stamped winner let lanes = st.lane_hi > 0l ? "\t{st.lane_lo}\t{st.lane_hi}" : "" var each = "" @@ -719,7 +719,7 @@ def main : int { } var fs <- formats_of(cfg.fmt) if (!cfg.tsv) { - print("kq_kernel_bench: n={cfg.n} d={cfg.d} ntok={cfg.ntok} rounds={cfg.rounds} team={cfg.team} tune_mode={get_env_variable("DAS_TUNE_MODE")}\n") + print("kq_kernel_bench: n={cfg.n} d={cfg.d} ntok={cfg.ntok} rounds={cfg.rounds} team={cfg.team} tune_mode={g_env_tune.tune_mode}\n") print("fmt\ttier\tperm\tn\td\tntok\tbest_us\tmedian_us\tns_per_sb\tGB/s\tGFLOP/s{cfg.team ? "\tlane_lo\tlane_hi" : ""}\n") } var rows = 0