From f642bed632dddbbefbb52b718474bd3343a2b534 Mon Sep 17 00:00:00 2001 From: CodSpeed Bot Date: Mon, 27 Jul 2026 03:49:42 +0000 Subject: [PATCH] ggml-cpu: fuse row-max scan into the mask pass in ggml_compute_forward_soft_max_f32 The CPU soft-max kernel did a separate ggml_vec_max_f32 scan over the per-row work buffer to find the row maximum for numerically-stable exponentiation, re-reading the entire row the mask pass had just written. When an additive mask is present (the common attention case), accumulate the row maximum inside the same loop that applies the mask, dropping one full pass over the row. The dedicated scan is kept only for the no-mask path. The fused reduction is identical to ggml_vec_max_f32 (same seed, update, order and values), so results are unchanged. Verified with test-backend-ops (212/212 SOFT_MAX cases pass on CPU). --- ggml/src/ggml-cpu/ops.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 42ec809ce521..c65e939b4d6f 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -5516,16 +5516,26 @@ static void ggml_compute_forward_soft_max_f32( ggml_vec_cpy_f32 (ne00, wp, sp); ggml_vec_scale_f32(ne00, wp, scale); + + // The additive mask is applied in a full pass over `wp`, so the + // row maximum (needed next for the numerically-stable soft-max) + // is accumulated in that same pass instead of re-reading the + // whole row afterwards with a separate ggml_vec_max_f32 scan. + float max = -INFINITY; if (mp_f32) { if (use_f16) { for (int i = 0; i < ne00; ++i) { wp[i] += slope*GGML_CPU_FP16_TO_FP32(mp_f16[i]); + max = MAX(max, wp[i]); } } else { for (int i = 0; i < ne00; ++i) { wp[i] += slope*mp_f32[i]; + max = MAX(max, wp[i]); } } + } else { + ggml_vec_max_f32(ne00, &max, wp); } #ifndef NDEBUG @@ -5535,9 +5545,6 @@ static void ggml_compute_forward_soft_max_f32( } #endif // NDEBUG - float max = -INFINITY; - ggml_vec_max_f32(ne00, &max, wp); - // if we have sinks, make a correction as if they were included in the softmax if (sk) { max = MAX(max, sk[i02]);