From 423c119c004b01786f36517a6db880586e02b7e6 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Sun, 13 Sep 2026 08:58:09 -0600 Subject: [PATCH] vulkan-shaders-gen: retry an empty compile, and never declare what is not defined A shader whose compile produces no SPIR-V becomes an undefined reference at link, twenty minutes later, naming a symbol in vendored code with nothing earlier in the log to explain it: ld.bfd: ggml-vulkan.cpp:3964: undefined reference to `matmul_id_subgroup_nvfp4_f32_aligned_f16acc_cm1_len' write_output_files() emits the declaration before reading the artefact, then skips the definition if the file is empty, silently. The symbol is left declared and never defined. Compile success was also judged by stderr alone, with no exit code, so a shader that merely warns is discarded while one that reports nothing and writes nothing is accepted. Success is now judged by the artefact: the SPIR-V must exist and be non-empty. An empty result is retried up to three times with short backoff before giving up. This has been seen in CI with no accompanying diagnostic and does not reproduce locally, so it appears environmental and rare; retrying costs milliseconds on a genuinely broken shader and saves a build that would otherwise fail at link for no visible reason. After the last attempt the shader is named and generation fails. Verified both ways against an injected failure. A transient -- first attempt empty, later attempts normal -- recovers and the build links. A permanent one reports: shader matmul_id_subgroup_nvfp4_f32_aligned_f16acc_cm1 produced no SPIR-V; retrying (2/3) shader matmul_id_subgroup_nvfp4_f32_aligned_f16acc_cm1 produced no SPIR-V; retrying (3/3) cannot compile matmul_id_subgroup_nvfp4_f32_aligned_f16acc_cm1 after 3 attempts shader generation failed; see errors above and stops at generation instead of at link. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EkxqpYvUbjCpRDnFiNiVfx --- .../vulkan-shaders/vulkan-shaders-gen.cpp | 66 ++++++++++++++++--- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 40c223470..319bb7e62 100644 --- a/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/external/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,9 @@ std::mutex lock; std::vector> shader_fnames; +// Set when a shader yields no SPIR-V, so the build stops at generation rather +// than at a link error that points nowhere useful. +bool generation_failed = false; std::locale c_locale("C"); std::string GLSLC = "glslc"; @@ -324,6 +328,14 @@ compile_count_guard acquire_compile_slot() { return compile_count_guard(&compile_count, &decrement_compile_count); } +// A shader is usable only if its SPIR-V exists and is non-empty. A zero-byte +// file is what an interrupted or silently-failed compile leaves behind. +static bool spv_is_usable(const std::string & path) { + std::error_code ec; + const auto size = std::filesystem::file_size(path, ec); + return !ec && size > 0; +} + void string_to_spv_func(std::string name, std::string in_path, std::string out_path, std::map defines, bool coopmat, bool dep_file, compile_count_guard slot) { std::string target_env = (name.find("_cm2") != std::string::npos) ? "--target-env=vulkan1.3" : "--target-env=vulkan1.2"; @@ -365,22 +377,49 @@ void string_to_spv_func(std::string name, std::string in_path, std::string out_p std::string stdout_str, stderr_str; try { - // std::cout << "Executing command: "; - // for (const auto& part : cmd) { - // std::cout << part << " "; - // } - // std::cout << std::endl; + // Success is judged by the artefact, not by stderr. Judging by stderr + // discards a shader over a warning, and misses the case that matters + // more: a compile that reports nothing and writes nothing. + // + // An empty result is retried rather than accepted. It has been seen in + // CI with no accompanying diagnostic, and retrying costs milliseconds + // on a genuinely broken shader while saving a build that would + // otherwise fail much later at link, naming a symbol whose absence has + // no visible cause. + constexpr int max_attempts = 3; + bool produced = false; + + for (int attempt = 1; attempt <= max_attempts && !produced; ++attempt) { + stdout_str.clear(); + stderr_str.clear(); + execute_command(cmd, stdout_str, stderr_str); + produced = spv_is_usable(out_path); + + if (!produced && attempt < max_attempts) { + std::cerr << "shader " << name << " produced no SPIR-V; retrying (" + << (attempt + 1) << "/" << max_attempts << ")" << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(100 * attempt)); + } + } - execute_command(cmd, stdout_str, stderr_str); - if (!stderr_str.empty()) { - std::cerr << "cannot compile " << name << "\n\n"; + if (!produced) { + std::cerr << "cannot compile " << name << " after " << max_attempts + << " attempts\n\n"; for (const auto& part : cmd) { std::cerr << part << " "; } std::cerr << "\n\n" << stderr_str << std::endl; + generation_failed = true; return; } + if (!stderr_str.empty()) { + // Diagnostics alongside a usable artefact are warnings. Keeping the + // shader is the point: dropping it is what leaves a declaration + // with no definition. + std::cerr << "warnings compiling " << name << ":\n" << stderr_str << std::endl; + } + if (dep_file) { // replace .spv output path with the embed .cpp path which is used as output in CMakeLists.txt std::string dep = read_binary_file(target_cpp + ".d", true); @@ -1050,6 +1089,12 @@ void write_output_files() { if (input_filepath != "") { std::string data = read_binary_file(path); if (data.empty()) { + // The declaration above is already written, so skipping the + // definition leaves a symbol declared and never defined, which + // surfaces much later as an undefined reference at link. + std::cerr << "ERROR: shader '" << name << "' produced no SPIR-V (" + << path << ")\n"; + generation_failed = true; continue; } @@ -1196,5 +1241,10 @@ int main(int argc, char** argv) { write_output_files(); + if (generation_failed) { + std::cerr << "shader generation failed; see errors above" << std::endl; + return EXIT_FAILURE; + } + return EXIT_SUCCESS; }