From 4854863f375e7f6a3d65ae6acfb4bf38ed231dac Mon Sep 17 00:00:00 2001 From: CodSpeed Bot Date: Mon, 27 Jul 2026 05:55:13 +0000 Subject: [PATCH] Vectorize ggml_vec_max_f32 with AVX2 for the soft-max row-max scan The scalar reduction in ggml_vec_max_f32 is a serial dependency the compiler does not auto-vectorize, dominating the soft-max row pass. Reduce eight lanes at a time with _mm256_max_ps and fold to a scalar, keeping a scalar tail and a scalar fallback for non-AVX2 targets. --- ggml/src/ggml-cpu/vec.h | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index 5de9cb5b7e09..030cf4357e80 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -1539,14 +1539,35 @@ inline static void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16 } inline static void ggml_vec_max_f32(const int n, float * s, const float * x) { -#ifndef GGML_USE_ACCELERATE +#if defined(GGML_USE_ACCELERATE) + vDSP_maxv(x, 1, s, n); +#elif defined(__AVX2__) + // Vectorized reduction: the scalar `max = MAX(max, x[i])` loop below is not + // auto-vectorized by the compiler (the reduction is serial), so it dominates + // the soft-max row pass. Reduce eight lanes at a time with a running vector + // max and fold to a scalar at the end. float max = -INFINITY; - for (int i = 0; i < n; ++i) { + int i = 0; + if (n >= 8) { + __m256 vmax = _mm256_set1_ps(-INFINITY); + for (; i + 7 < n; i += 8) { + vmax = _mm256_max_ps(vmax, _mm256_loadu_ps(x + i)); + } + __m128 v = _mm_max_ps(_mm256_castps256_ps128(vmax), _mm256_extractf128_ps(vmax, 1)); + v = _mm_max_ps(v, _mm_movehl_ps(v, v)); + v = _mm_max_ss(v, _mm_movehdup_ps(v)); + max = _mm_cvtss_f32(v); + } + for (; i < n; ++i) { max = MAX(max, x[i]); } *s = max; #else - vDSP_maxv(x, 1, s, n); + float max = -INFINITY; + for (int i = 0; i < n; ++i) { + max = MAX(max, x[i]); + } + *s = max; #endif }