Skip to content

breeze: enable bf16 activation rounding on Metal #554

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gqf2008 wants to merge 2 commits into
base: main
Choose a base branch
from
+14 −9
Open

breeze: enable bf16 activation rounding on Metal #554

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fe6946e
ggml-metal: add bf16<->f16 copy kernels
gqf2008 Sep 15, 2026
00d2937
breeze: enable bf16 activation rounding on Metal
gqf2008 Sep 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions external/ggml/src/ggml-metal/ggml-metal-device.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1279,14 +1279,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
switch (op->type) {
case GGML_TYPE_F32:
case GGML_TYPE_F16:
return true;
case GGML_TYPE_BF16: return true;
default:
return false;
}
case GGML_TYPE_BF16:
switch (op->type) {
case GGML_TYPE_F32:
case GGML_TYPE_BF16:
case GGML_TYPE_F16: case GGML_TYPE_BF16:
return true;
default:
return false;
Expand Down
4 changes: 4 additions & 0 deletions external/ggml/src/ggml-metal/ggml-metal.metal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7932,6 +7932,8 @@ template [[host_name("kernel_cpy_contig_f16_f16")]] kernel kernel_cpy_contig_t k
#if defined(GGML_METAL_HAS_BF16)
template [[host_name("kernel_cpy_contig_bf16_f32")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t<bfloat, float>;
template [[host_name("kernel_cpy_contig_bf16_bf16")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t<bfloat, bfloat>;
template [[host_name("kernel_cpy_contig_f16_bf16")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t<half, bfloat>;
template [[host_name("kernel_cpy_contig_bf16_f16")]] kernel kernel_cpy_contig_t kernel_cpy_contig_t_t<bfloat, half>;
#endif

template<typename T>
Expand Down Expand Up @@ -8064,6 +8066,8 @@ template [[host_name("kernel_cpy_f16_f16")]] kernel kernel_cpy_t kernel_cpy_t_
#if defined(GGML_METAL_HAS_BF16)
template [[host_name("kernel_cpy_bf16_f32")]] kernel kernel_cpy_t kernel_cpy_t_t<bfloat, float>;
template [[host_name("kernel_cpy_bf16_bf16")]] kernel kernel_cpy_t kernel_cpy_t_t<bfloat, bfloat>;
template [[host_name("kernel_cpy_f16_bf16")]] kernel kernel_cpy_t kernel_cpy_t_t<half, bfloat>;
template [[host_name("kernel_cpy_bf16_f16")]] kernel kernel_cpy_t kernel_cpy_t_t<bfloat, half>;
#endif

template<short QK,
Expand Down
15 changes: 8 additions & 7 deletions src/models/breeze_tts/generator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,13 @@ struct GgmlContextDeleter {
modules::QwenDecoderActivationCastPolicy breeze_bf16_activation_policy(core::BackendType backend_type) {
modules::QwenDecoderActivationCastPolicy policy;
if (backend_type != core::BackendType::Cuda && backend_type != core::BackendType::Hip &&
backend_type != core::BackendType::Vulkan) {
backend_type != core::BackendType::Vulkan && backend_type != core::BackendType::Metal) {
return policy;
}
policy.enabled = true;
policy.type = GGML_TYPE_BF16;
// CUDA/HIP/Vulkan implement the fused round-to-bf16 unary op.
policy.fused_round = backend_type == core::BackendType::Cuda || backend_type == core::BackendType::Hip ||
backend_type == core::BackendType::Vulkan;
policy.fused_round = backend_type != core::BackendType::Metal;
policy.after_input_norm = true;
policy.after_qkv_projection = true;
policy.after_qk_norm = true;
Expand Down Expand Up @@ -114,12 +113,13 @@ modules::QwenCausalDecodeRuntimeConfig backbone_config(
out.decoder.stack.runtime.static_cache.update_mode = modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows;
out.decoder.stack.runtime.static_cache.set_rows_mode = modules::QwenDecoderStaticCacheSetRowsMode::BackendViewOptimized;
if (backend_type == core::BackendType::Cuda || backend_type == core::BackendType::Hip ||
backend_type == core::BackendType::Vulkan) {
backend_type == core::BackendType::Vulkan || backend_type == core::BackendType::Metal) {
// BF16 KV cache matches the reference implementation, but flash
// attention only accelerates bf16 cache with native bf16 MMA
// (sm_80+); on older parts it is ~3x slower, so only HIP uses it.
out.decoder.static_cache_type =
backend_type == core::BackendType::Hip ? GGML_TYPE_BF16 : GGML_TYPE_F16;
(backend_type == core::BackendType::Hip || backend_type == core::BackendType::Metal)
? GGML_TYPE_BF16 : GGML_TYPE_F16;
out.decoder.stack.activation_cast = breeze_bf16_activation_policy(backend_type);
}
out.decoder.logits_size = config.lm_head_size;
Expand Down Expand Up @@ -168,11 +168,12 @@ modules::QwenCausalDecodeRuntimeConfig depth_config(
out.decoder.stack.runtime.static_cache.update_mode = modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows;
out.decoder.stack.runtime.static_cache.set_rows_mode = modules::QwenDecoderStaticCacheSetRowsMode::BackendViewOptimized;
if (backend_type == core::BackendType::Cuda || backend_type == core::BackendType::Hip ||
backend_type == core::BackendType::Vulkan) {
backend_type == core::BackendType::Vulkan || backend_type == core::BackendType::Metal) {
// See backbone_config: only HIP uses a bf16 KV cache; CUDA and Vulkan
// keep F16.
out.decoder.static_cache_type =
backend_type == core::BackendType::Hip ? GGML_TYPE_BF16 : GGML_TYPE_F16;
(backend_type == core::BackendType::Hip || backend_type == core::BackendType::Metal)
? GGML_TYPE_BF16 : GGML_TYPE_F16;
out.decoder.stack.activation_cast = breeze_bf16_activation_policy(backend_type);
}
out.decoder.logits_mode = modules::QwenCausalDecoderLogitsMode::LastStep;
Expand Down
Loading