From 64adf7e4fdaed258b28a2114ede80fd58cfa0d3d Mon Sep 17 00:00:00 2001 From: CodSpeed Bot Date: Tue, 28 Jul 2026 05:24:22 +0000 Subject: [PATCH] ggml-cpu: unroll ggml_vec_dot_f16 into four accumulator groups Process 4*GGML_F16_STEP elements per iteration using four independent accumulator groups in the generic GGML_SIMD path of ggml_vec_dot_f16. This cuts loop iterations (and their counter/branch/index arithmetic) by 4x for the same FMA work while keeping more independent FMA chains in flight to hide FMA latency. The groups are combined before the existing horizontal reduce, so the result matches the original accumulation. A tail loop handles the remaining full GGML_F16_STEP block and the scalar leftover loop is unchanged. The SVE and RISC-V paths are untouched. --- ggml/src/ggml-cpu/vec.cpp | 47 +++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp index ff2b636df86c..cd2c1798f820 100644 --- a/ggml/src/ggml-cpu/vec.cpp +++ b/ggml/src/ggml-cpu/vec.cpp @@ -342,14 +342,53 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G } #endif // __riscv_zvfh #else - const int np = (n & ~(GGML_F16_STEP - 1)); - - GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO }; + // Process 4*GGML_F16_STEP elements per iteration using four independent + // accumulator groups. Compared to the original single-group loop this + // cuts the number of loop iterations (and their counter/branch/index + // arithmetic) by 4x for the same amount of FMA work, while keeping many + // more independent FMA chains in flight to hide the FMA latency. The + // groups are summed together before the final horizontal reduce, so the + // result matches the original single-group accumulation. + const int np4 = (n & ~(4*GGML_F16_STEP - 1)); + + GGML_F16_VEC sum [GGML_F16_ARR] = { GGML_F16_VEC_ZERO }; + GGML_F16_VEC sum2[GGML_F16_ARR] = { GGML_F16_VEC_ZERO }; + GGML_F16_VEC sum3[GGML_F16_ARR] = { GGML_F16_VEC_ZERO }; + GGML_F16_VEC sum4[GGML_F16_ARR] = { GGML_F16_VEC_ZERO }; GGML_F16_VEC ax[GGML_F16_ARR]; GGML_F16_VEC ay[GGML_F16_ARR]; - for (int i = 0; i < np; i += GGML_F16_STEP) { + for (int i = 0; i < np4; i += 4*GGML_F16_STEP) { + for (int j = 0; j < GGML_F16_ARR; j++) { + ax[j] = GGML_F16_VEC_LOAD(x + i + 0*GGML_F16_STEP + j*GGML_F16_EPR, j); + ay[j] = GGML_F16_VEC_LOAD(y + i + 0*GGML_F16_STEP + j*GGML_F16_EPR, j); + sum[j] = GGML_F16_VEC_FMA(sum[j], ax[j], ay[j]); + + ax[j] = GGML_F16_VEC_LOAD(x + i + 1*GGML_F16_STEP + j*GGML_F16_EPR, j); + ay[j] = GGML_F16_VEC_LOAD(y + i + 1*GGML_F16_STEP + j*GGML_F16_EPR, j); + sum2[j] = GGML_F16_VEC_FMA(sum2[j], ax[j], ay[j]); + + ax[j] = GGML_F16_VEC_LOAD(x + i + 2*GGML_F16_STEP + j*GGML_F16_EPR, j); + ay[j] = GGML_F16_VEC_LOAD(y + i + 2*GGML_F16_STEP + j*GGML_F16_EPR, j); + sum3[j] = GGML_F16_VEC_FMA(sum3[j], ax[j], ay[j]); + + ax[j] = GGML_F16_VEC_LOAD(x + i + 3*GGML_F16_STEP + j*GGML_F16_EPR, j); + ay[j] = GGML_F16_VEC_LOAD(y + i + 3*GGML_F16_STEP + j*GGML_F16_EPR, j); + sum4[j] = GGML_F16_VEC_FMA(sum4[j], ax[j], ay[j]); + } + } + + // combine the four accumulator groups + for (int j = 0; j < GGML_F16_ARR; j++) { + sum [j] = GGML_F16_VEC_ADD(sum [j], sum3[j]); + sum2[j] = GGML_F16_VEC_ADD(sum2[j], sum4[j]); + sum [j] = GGML_F16_VEC_ADD(sum [j], sum2[j]); + } + + // handle the remaining full GGML_F16_STEP blocks + const int np = (n & ~(GGML_F16_STEP - 1)); + for (int i = np4; i < np; i += GGML_F16_STEP) { for (int j = 0; j < GGML_F16_ARR; j++) { ax[j] = GGML_F16_VEC_LOAD(x + i + j*GGML_F16_EPR, j); ay[j] = GGML_F16_VEC_LOAD(y + i + j*GGML_F16_EPR, j);